1 //===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This contains code to emit Builtin calls as LLVM code. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "CodeGenFunction.h" 15 #include "CGCXXABI.h" 16 #include "CGObjCRuntime.h" 17 #include "CodeGenModule.h" 18 #include "TargetInfo.h" 19 #include "clang/AST/ASTContext.h" 20 #include "clang/AST/Decl.h" 21 #include "clang/Basic/TargetBuiltins.h" 22 #include "clang/Basic/TargetInfo.h" 23 #include "clang/CodeGen/CGFunctionInfo.h" 24 #include "llvm/ADT/StringExtras.h" 25 #include "llvm/IR/CallSite.h" 26 #include "llvm/IR/DataLayout.h" 27 #include "llvm/IR/InlineAsm.h" 28 #include "llvm/IR/Intrinsics.h" 29 #include <sstream> 30 31 using namespace clang; 32 using namespace CodeGen; 33 using namespace llvm; 34 35 /// getBuiltinLibFunction - Given a builtin id for a function like 36 /// "__builtin_fabsf", return a Function* for "fabsf". 37 llvm::Value *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD, 38 unsigned BuiltinID) { 39 assert(Context.BuiltinInfo.isLibFunction(BuiltinID)); 40 41 // Get the name, skip over the __builtin_ prefix (if necessary). 42 StringRef Name; 43 GlobalDecl D(FD); 44 45 // If the builtin has been declared explicitly with an assembler label, 46 // use the mangled name. This differs from the plain label on platforms 47 // that prefix labels. 48 if (FD->hasAttr<AsmLabelAttr>()) 49 Name = getMangledName(D); 50 else 51 Name = Context.BuiltinInfo.getName(BuiltinID) + 10; 52 53 llvm::FunctionType *Ty = 54 cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType())); 55 56 return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false); 57 } 58 59 /// Emit the conversions required to turn the given value into an 60 /// integer of the given size. 61 static Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V, 62 QualType T, llvm::IntegerType *IntType) { 63 V = CGF.EmitToMemory(V, T); 64 65 if (V->getType()->isPointerTy()) 66 return CGF.Builder.CreatePtrToInt(V, IntType); 67 68 assert(V->getType() == IntType); 69 return V; 70 } 71 72 static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, 73 QualType T, llvm::Type *ResultType) { 74 V = CGF.EmitFromMemory(V, T); 75 76 if (ResultType->isPointerTy()) 77 return CGF.Builder.CreateIntToPtr(V, ResultType); 78 79 assert(V->getType() == ResultType); 80 return V; 81 } 82 83 /// Utility to insert an atomic instruction based on Instrinsic::ID 84 /// and the expression node. 85 static Value *MakeBinaryAtomicValue(CodeGenFunction &CGF, 86 llvm::AtomicRMWInst::BinOp Kind, 87 const CallExpr *E) { 88 QualType T = E->getType(); 89 assert(E->getArg(0)->getType()->isPointerType()); 90 assert(CGF.getContext().hasSameUnqualifiedType(T, 91 E->getArg(0)->getType()->getPointeeType())); 92 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType())); 93 94 llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); 95 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 96 97 llvm::IntegerType *IntType = 98 llvm::IntegerType::get(CGF.getLLVMContext(), 99 CGF.getContext().getTypeSize(T)); 100 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 101 102 llvm::Value *Args[2]; 103 Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); 104 Args[1] = CGF.EmitScalarExpr(E->getArg(1)); 105 llvm::Type *ValueType = Args[1]->getType(); 106 Args[1] = EmitToInt(CGF, Args[1], T, IntType); 107 108 llvm::Value *Result = CGF.Builder.CreateAtomicRMW( 109 Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent); 110 return EmitFromInt(CGF, Result, T, ValueType); 111 } 112 113 static Value *EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) { 114 Value *Val = CGF.EmitScalarExpr(E->getArg(0)); 115 Value *Address = CGF.EmitScalarExpr(E->getArg(1)); 116 117 // Convert the type of the pointer to a pointer to the stored type. 118 Val = CGF.EmitToMemory(Val, E->getArg(0)->getType()); 119 Value *BC = CGF.Builder.CreateBitCast( 120 Address, llvm::PointerType::getUnqual(Val->getType()), "cast"); 121 LValue LV = CGF.MakeNaturalAlignAddrLValue(BC, E->getArg(0)->getType()); 122 LV.setNontemporal(true); 123 CGF.EmitStoreOfScalar(Val, LV, false); 124 return nullptr; 125 } 126 127 static Value *EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) { 128 Value *Address = CGF.EmitScalarExpr(E->getArg(0)); 129 130 LValue LV = CGF.MakeNaturalAlignAddrLValue(Address, E->getType()); 131 LV.setNontemporal(true); 132 return CGF.EmitLoadOfScalar(LV, E->getExprLoc()); 133 } 134 135 static RValue EmitBinaryAtomic(CodeGenFunction &CGF, 136 llvm::AtomicRMWInst::BinOp Kind, 137 const CallExpr *E) { 138 return RValue::get(MakeBinaryAtomicValue(CGF, Kind, E)); 139 } 140 141 /// Utility to insert an atomic instruction based Instrinsic::ID and 142 /// the expression node, where the return value is the result of the 143 /// operation. 144 static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, 145 llvm::AtomicRMWInst::BinOp Kind, 146 const CallExpr *E, 147 Instruction::BinaryOps Op, 148 bool Invert = false) { 149 QualType T = E->getType(); 150 assert(E->getArg(0)->getType()->isPointerType()); 151 assert(CGF.getContext().hasSameUnqualifiedType(T, 152 E->getArg(0)->getType()->getPointeeType())); 153 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType())); 154 155 llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); 156 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 157 158 llvm::IntegerType *IntType = 159 llvm::IntegerType::get(CGF.getLLVMContext(), 160 CGF.getContext().getTypeSize(T)); 161 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 162 163 llvm::Value *Args[2]; 164 Args[1] = CGF.EmitScalarExpr(E->getArg(1)); 165 llvm::Type *ValueType = Args[1]->getType(); 166 Args[1] = EmitToInt(CGF, Args[1], T, IntType); 167 Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); 168 169 llvm::Value *Result = CGF.Builder.CreateAtomicRMW( 170 Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent); 171 Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]); 172 if (Invert) 173 Result = CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result, 174 llvm::ConstantInt::get(IntType, -1)); 175 Result = EmitFromInt(CGF, Result, T, ValueType); 176 return RValue::get(Result); 177 } 178 179 /// @brief Utility to insert an atomic cmpxchg instruction. 180 /// 181 /// @param CGF The current codegen function. 182 /// @param E Builtin call expression to convert to cmpxchg. 183 /// arg0 - address to operate on 184 /// arg1 - value to compare with 185 /// arg2 - new value 186 /// @param ReturnBool Specifies whether to return success flag of 187 /// cmpxchg result or the old value. 188 /// 189 /// @returns result of cmpxchg, according to ReturnBool 190 static Value *MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, 191 bool ReturnBool) { 192 QualType T = ReturnBool ? E->getArg(1)->getType() : E->getType(); 193 llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); 194 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 195 196 llvm::IntegerType *IntType = llvm::IntegerType::get( 197 CGF.getLLVMContext(), CGF.getContext().getTypeSize(T)); 198 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 199 200 Value *Args[3]; 201 Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); 202 Args[1] = CGF.EmitScalarExpr(E->getArg(1)); 203 llvm::Type *ValueType = Args[1]->getType(); 204 Args[1] = EmitToInt(CGF, Args[1], T, IntType); 205 Args[2] = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType); 206 207 Value *Pair = CGF.Builder.CreateAtomicCmpXchg( 208 Args[0], Args[1], Args[2], llvm::AtomicOrdering::SequentiallyConsistent, 209 llvm::AtomicOrdering::SequentiallyConsistent); 210 if (ReturnBool) 211 // Extract boolean success flag and zext it to int. 212 return CGF.Builder.CreateZExt(CGF.Builder.CreateExtractValue(Pair, 1), 213 CGF.ConvertType(E->getType())); 214 else 215 // Extract old value and emit it using the same type as compare value. 216 return EmitFromInt(CGF, CGF.Builder.CreateExtractValue(Pair, 0), T, 217 ValueType); 218 } 219 220 /// EmitFAbs - Emit a call to @llvm.fabs(). 221 static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) { 222 Value *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType()); 223 llvm::CallInst *Call = CGF.Builder.CreateCall(F, V); 224 Call->setDoesNotAccessMemory(); 225 return Call; 226 } 227 228 /// Emit the computation of the sign bit for a floating point value. Returns 229 /// the i1 sign bit value. 230 static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) { 231 LLVMContext &C = CGF.CGM.getLLVMContext(); 232 233 llvm::Type *Ty = V->getType(); 234 int Width = Ty->getPrimitiveSizeInBits(); 235 llvm::Type *IntTy = llvm::IntegerType::get(C, Width); 236 V = CGF.Builder.CreateBitCast(V, IntTy); 237 if (Ty->isPPC_FP128Ty()) { 238 // We want the sign bit of the higher-order double. The bitcast we just 239 // did works as if the double-double was stored to memory and then 240 // read as an i128. The "store" will put the higher-order double in the 241 // lower address in both little- and big-Endian modes, but the "load" 242 // will treat those bits as a different part of the i128: the low bits in 243 // little-Endian, the high bits in big-Endian. Therefore, on big-Endian 244 // we need to shift the high bits down to the low before truncating. 245 Width >>= 1; 246 if (CGF.getTarget().isBigEndian()) { 247 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width); 248 V = CGF.Builder.CreateLShr(V, ShiftCst); 249 } 250 // We are truncating value in order to extract the higher-order 251 // double, which we will be using to extract the sign from. 252 IntTy = llvm::IntegerType::get(C, Width); 253 V = CGF.Builder.CreateTrunc(V, IntTy); 254 } 255 Value *Zero = llvm::Constant::getNullValue(IntTy); 256 return CGF.Builder.CreateICmpSLT(V, Zero); 257 } 258 259 static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *Fn, 260 const CallExpr *E, llvm::Value *calleeValue) { 261 return CGF.EmitCall(E->getCallee()->getType(), calleeValue, E, 262 ReturnValueSlot(), Fn); 263 } 264 265 /// \brief Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.* 266 /// depending on IntrinsicID. 267 /// 268 /// \arg CGF The current codegen function. 269 /// \arg IntrinsicID The ID for the Intrinsic we wish to generate. 270 /// \arg X The first argument to the llvm.*.with.overflow.*. 271 /// \arg Y The second argument to the llvm.*.with.overflow.*. 272 /// \arg Carry The carry returned by the llvm.*.with.overflow.*. 273 /// \returns The result (i.e. sum/product) returned by the intrinsic. 274 static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF, 275 const llvm::Intrinsic::ID IntrinsicID, 276 llvm::Value *X, llvm::Value *Y, 277 llvm::Value *&Carry) { 278 // Make sure we have integers of the same width. 279 assert(X->getType() == Y->getType() && 280 "Arguments must be the same type. (Did you forget to make sure both " 281 "arguments have the same integer width?)"); 282 283 llvm::Value *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType()); 284 llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, {X, Y}); 285 Carry = CGF.Builder.CreateExtractValue(Tmp, 1); 286 return CGF.Builder.CreateExtractValue(Tmp, 0); 287 } 288 289 // Emit a simple mangled intrinsic that has 1 argument and a return type 290 // matching the argument type. 291 static Value *emitUnaryBuiltin(CodeGenFunction &CGF, 292 const CallExpr *E, 293 unsigned IntrinsicID) { 294 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); 295 296 Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); 297 return CGF.Builder.CreateCall(F, Src0); 298 } 299 300 // Emit an intrinsic that has 3 float or double operands. 301 static Value *emitTernaryFPBuiltin(CodeGenFunction &CGF, 302 const CallExpr *E, 303 unsigned IntrinsicID) { 304 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); 305 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); 306 llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2)); 307 308 Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); 309 return CGF.Builder.CreateCall(F, {Src0, Src1, Src2}); 310 } 311 312 // Emit an intrinsic that has 1 float or double operand, and 1 integer. 313 static Value *emitFPIntBuiltin(CodeGenFunction &CGF, 314 const CallExpr *E, 315 unsigned IntrinsicID) { 316 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); 317 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); 318 319 Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); 320 return CGF.Builder.CreateCall(F, {Src0, Src1}); 321 } 322 323 namespace { 324 struct WidthAndSignedness { 325 unsigned Width; 326 bool Signed; 327 }; 328 } 329 330 static WidthAndSignedness 331 getIntegerWidthAndSignedness(const clang::ASTContext &context, 332 const clang::QualType Type) { 333 assert(Type->isIntegerType() && "Given type is not an integer."); 334 unsigned Width = Type->isBooleanType() ? 1 : context.getTypeInfo(Type).Width; 335 bool Signed = Type->isSignedIntegerType(); 336 return {Width, Signed}; 337 } 338 339 // Given one or more integer types, this function produces an integer type that 340 // encompasses them: any value in one of the given types could be expressed in 341 // the encompassing type. 342 static struct WidthAndSignedness 343 EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) { 344 assert(Types.size() > 0 && "Empty list of types."); 345 346 // If any of the given types is signed, we must return a signed type. 347 bool Signed = false; 348 for (const auto &Type : Types) { 349 Signed |= Type.Signed; 350 } 351 352 // The encompassing type must have a width greater than or equal to the width 353 // of the specified types. Aditionally, if the encompassing type is signed, 354 // its width must be strictly greater than the width of any unsigned types 355 // given. 356 unsigned Width = 0; 357 for (const auto &Type : Types) { 358 unsigned MinWidth = Type.Width + (Signed && !Type.Signed); 359 if (Width < MinWidth) { 360 Width = MinWidth; 361 } 362 } 363 364 return {Width, Signed}; 365 } 366 367 Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) { 368 llvm::Type *DestType = Int8PtrTy; 369 if (ArgValue->getType() != DestType) 370 ArgValue = 371 Builder.CreateBitCast(ArgValue, DestType, ArgValue->getName().data()); 372 373 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend; 374 return Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue); 375 } 376 377 /// Checks if using the result of __builtin_object_size(p, @p From) in place of 378 /// __builtin_object_size(p, @p To) is correct 379 static bool areBOSTypesCompatible(int From, int To) { 380 // Note: Our __builtin_object_size implementation currently treats Type=0 and 381 // Type=2 identically. Encoding this implementation detail here may make 382 // improving __builtin_object_size difficult in the future, so it's omitted. 383 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2); 384 } 385 386 static llvm::Value * 387 getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) { 388 return ConstantInt::get(ResType, (Type & 2) ? 0 : -1, /*isSigned=*/true); 389 } 390 391 llvm::Value * 392 CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type, 393 llvm::IntegerType *ResType) { 394 uint64_t ObjectSize; 395 if (!E->tryEvaluateObjectSize(ObjectSize, getContext(), Type)) 396 return emitBuiltinObjectSize(E, Type, ResType); 397 return ConstantInt::get(ResType, ObjectSize, /*isSigned=*/true); 398 } 399 400 /// Returns a Value corresponding to the size of the given expression. 401 /// This Value may be either of the following: 402 /// - A llvm::Argument (if E is a param with the pass_object_size attribute on 403 /// it) 404 /// - A call to the @llvm.objectsize intrinsic 405 llvm::Value * 406 CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type, 407 llvm::IntegerType *ResType) { 408 // We need to reference an argument if the pointer is a parameter with the 409 // pass_object_size attribute. 410 if (auto *D = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts())) { 411 auto *Param = dyn_cast<ParmVarDecl>(D->getDecl()); 412 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>(); 413 if (Param != nullptr && PS != nullptr && 414 areBOSTypesCompatible(PS->getType(), Type)) { 415 auto Iter = SizeArguments.find(Param); 416 assert(Iter != SizeArguments.end()); 417 418 const ImplicitParamDecl *D = Iter->second; 419 auto DIter = LocalDeclMap.find(D); 420 assert(DIter != LocalDeclMap.end()); 421 422 return EmitLoadOfScalar(DIter->second, /*volatile=*/false, 423 getContext().getSizeType(), E->getLocStart()); 424 } 425 } 426 427 // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't 428 // evaluate E for side-effects. In either case, we shouldn't lower to 429 // @llvm.objectsize. 430 if (Type == 3 || E->HasSideEffects(getContext())) 431 return getDefaultBuiltinObjectSizeResult(Type, ResType); 432 433 // LLVM only supports 0 and 2, make sure that we pass along that 434 // as a boolean. 435 auto *CI = ConstantInt::get(Builder.getInt1Ty(), (Type & 2) >> 1); 436 // FIXME: Get right address space. 437 llvm::Type *Tys[] = {ResType, Builder.getInt8PtrTy(0)}; 438 Value *F = CGM.getIntrinsic(Intrinsic::objectsize, Tys); 439 return Builder.CreateCall(F, {EmitScalarExpr(E), CI}); 440 } 441 442 RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, 443 unsigned BuiltinID, const CallExpr *E, 444 ReturnValueSlot ReturnValue) { 445 // See if we can constant fold this builtin. If so, don't emit it at all. 446 Expr::EvalResult Result; 447 if (E->EvaluateAsRValue(Result, CGM.getContext()) && 448 !Result.hasSideEffects()) { 449 if (Result.Val.isInt()) 450 return RValue::get(llvm::ConstantInt::get(getLLVMContext(), 451 Result.Val.getInt())); 452 if (Result.Val.isFloat()) 453 return RValue::get(llvm::ConstantFP::get(getLLVMContext(), 454 Result.Val.getFloat())); 455 } 456 457 switch (BuiltinID) { 458 default: break; // Handle intrinsics and libm functions below. 459 case Builtin::BI__builtin___CFStringMakeConstantString: 460 case Builtin::BI__builtin___NSStringMakeConstantString: 461 return RValue::get(CGM.EmitConstantExpr(E, E->getType(), nullptr)); 462 case Builtin::BI__builtin_stdarg_start: 463 case Builtin::BI__builtin_va_start: 464 case Builtin::BI__va_start: 465 case Builtin::BI__builtin_va_end: 466 return RValue::get( 467 EmitVAStartEnd(BuiltinID == Builtin::BI__va_start 468 ? EmitScalarExpr(E->getArg(0)) 469 : EmitVAListRef(E->getArg(0)).getPointer(), 470 BuiltinID != Builtin::BI__builtin_va_end)); 471 case Builtin::BI__builtin_va_copy: { 472 Value *DstPtr = EmitVAListRef(E->getArg(0)).getPointer(); 473 Value *SrcPtr = EmitVAListRef(E->getArg(1)).getPointer(); 474 475 llvm::Type *Type = Int8PtrTy; 476 477 DstPtr = Builder.CreateBitCast(DstPtr, Type); 478 SrcPtr = Builder.CreateBitCast(SrcPtr, Type); 479 return RValue::get(Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy), 480 {DstPtr, SrcPtr})); 481 } 482 case Builtin::BI__builtin_abs: 483 case Builtin::BI__builtin_labs: 484 case Builtin::BI__builtin_llabs: { 485 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 486 487 Value *NegOp = Builder.CreateNeg(ArgValue, "neg"); 488 Value *CmpResult = 489 Builder.CreateICmpSGE(ArgValue, 490 llvm::Constant::getNullValue(ArgValue->getType()), 491 "abscond"); 492 Value *Result = 493 Builder.CreateSelect(CmpResult, ArgValue, NegOp, "abs"); 494 495 return RValue::get(Result); 496 } 497 case Builtin::BI__builtin_fabs: 498 case Builtin::BI__builtin_fabsf: 499 case Builtin::BI__builtin_fabsl: { 500 Value *Arg1 = EmitScalarExpr(E->getArg(0)); 501 Value *Result = EmitFAbs(*this, Arg1); 502 return RValue::get(Result); 503 } 504 case Builtin::BI__builtin_fmod: 505 case Builtin::BI__builtin_fmodf: 506 case Builtin::BI__builtin_fmodl: { 507 Value *Arg1 = EmitScalarExpr(E->getArg(0)); 508 Value *Arg2 = EmitScalarExpr(E->getArg(1)); 509 Value *Result = Builder.CreateFRem(Arg1, Arg2, "fmod"); 510 return RValue::get(Result); 511 } 512 513 case Builtin::BI__builtin_conj: 514 case Builtin::BI__builtin_conjf: 515 case Builtin::BI__builtin_conjl: { 516 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 517 Value *Real = ComplexVal.first; 518 Value *Imag = ComplexVal.second; 519 Value *Zero = 520 Imag->getType()->isFPOrFPVectorTy() 521 ? llvm::ConstantFP::getZeroValueForNegation(Imag->getType()) 522 : llvm::Constant::getNullValue(Imag->getType()); 523 524 Imag = Builder.CreateFSub(Zero, Imag, "sub"); 525 return RValue::getComplex(std::make_pair(Real, Imag)); 526 } 527 case Builtin::BI__builtin_creal: 528 case Builtin::BI__builtin_crealf: 529 case Builtin::BI__builtin_creall: 530 case Builtin::BIcreal: 531 case Builtin::BIcrealf: 532 case Builtin::BIcreall: { 533 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 534 return RValue::get(ComplexVal.first); 535 } 536 537 case Builtin::BI__builtin_cimag: 538 case Builtin::BI__builtin_cimagf: 539 case Builtin::BI__builtin_cimagl: 540 case Builtin::BIcimag: 541 case Builtin::BIcimagf: 542 case Builtin::BIcimagl: { 543 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 544 return RValue::get(ComplexVal.second); 545 } 546 547 case Builtin::BI__builtin_ctzs: 548 case Builtin::BI__builtin_ctz: 549 case Builtin::BI__builtin_ctzl: 550 case Builtin::BI__builtin_ctzll: { 551 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 552 553 llvm::Type *ArgType = ArgValue->getType(); 554 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); 555 556 llvm::Type *ResultType = ConvertType(E->getType()); 557 Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); 558 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef}); 559 if (Result->getType() != ResultType) 560 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 561 "cast"); 562 return RValue::get(Result); 563 } 564 case Builtin::BI__builtin_clzs: 565 case Builtin::BI__builtin_clz: 566 case Builtin::BI__builtin_clzl: 567 case Builtin::BI__builtin_clzll: { 568 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 569 570 llvm::Type *ArgType = ArgValue->getType(); 571 Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType); 572 573 llvm::Type *ResultType = ConvertType(E->getType()); 574 Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); 575 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef}); 576 if (Result->getType() != ResultType) 577 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 578 "cast"); 579 return RValue::get(Result); 580 } 581 case Builtin::BI__builtin_ffs: 582 case Builtin::BI__builtin_ffsl: 583 case Builtin::BI__builtin_ffsll: { 584 // ffs(x) -> x ? cttz(x) + 1 : 0 585 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 586 587 llvm::Type *ArgType = ArgValue->getType(); 588 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); 589 590 llvm::Type *ResultType = ConvertType(E->getType()); 591 Value *Tmp = 592 Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}), 593 llvm::ConstantInt::get(ArgType, 1)); 594 Value *Zero = llvm::Constant::getNullValue(ArgType); 595 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero"); 596 Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs"); 597 if (Result->getType() != ResultType) 598 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 599 "cast"); 600 return RValue::get(Result); 601 } 602 case Builtin::BI__builtin_parity: 603 case Builtin::BI__builtin_parityl: 604 case Builtin::BI__builtin_parityll: { 605 // parity(x) -> ctpop(x) & 1 606 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 607 608 llvm::Type *ArgType = ArgValue->getType(); 609 Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType); 610 611 llvm::Type *ResultType = ConvertType(E->getType()); 612 Value *Tmp = Builder.CreateCall(F, ArgValue); 613 Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1)); 614 if (Result->getType() != ResultType) 615 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 616 "cast"); 617 return RValue::get(Result); 618 } 619 case Builtin::BI__builtin_popcount: 620 case Builtin::BI__builtin_popcountl: 621 case Builtin::BI__builtin_popcountll: { 622 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 623 624 llvm::Type *ArgType = ArgValue->getType(); 625 Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType); 626 627 llvm::Type *ResultType = ConvertType(E->getType()); 628 Value *Result = Builder.CreateCall(F, ArgValue); 629 if (Result->getType() != ResultType) 630 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 631 "cast"); 632 return RValue::get(Result); 633 } 634 case Builtin::BI__builtin_unpredictable: { 635 // Always return the argument of __builtin_unpredictable. LLVM does not 636 // handle this builtin. Metadata for this builtin should be added directly 637 // to instructions such as branches or switches that use it. 638 return RValue::get(EmitScalarExpr(E->getArg(0))); 639 } 640 case Builtin::BI__builtin_expect: { 641 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 642 llvm::Type *ArgType = ArgValue->getType(); 643 644 Value *ExpectedValue = EmitScalarExpr(E->getArg(1)); 645 // Don't generate llvm.expect on -O0 as the backend won't use it for 646 // anything. 647 // Note, we still IRGen ExpectedValue because it could have side-effects. 648 if (CGM.getCodeGenOpts().OptimizationLevel == 0) 649 return RValue::get(ArgValue); 650 651 Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType); 652 Value *Result = 653 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue}, "expval"); 654 return RValue::get(Result); 655 } 656 case Builtin::BI__builtin_assume_aligned: { 657 Value *PtrValue = EmitScalarExpr(E->getArg(0)); 658 Value *OffsetValue = 659 (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : nullptr; 660 661 Value *AlignmentValue = EmitScalarExpr(E->getArg(1)); 662 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue); 663 unsigned Alignment = (unsigned) AlignmentCI->getZExtValue(); 664 665 EmitAlignmentAssumption(PtrValue, Alignment, OffsetValue); 666 return RValue::get(PtrValue); 667 } 668 case Builtin::BI__assume: 669 case Builtin::BI__builtin_assume: { 670 if (E->getArg(0)->HasSideEffects(getContext())) 671 return RValue::get(nullptr); 672 673 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 674 Value *FnAssume = CGM.getIntrinsic(Intrinsic::assume); 675 return RValue::get(Builder.CreateCall(FnAssume, ArgValue)); 676 } 677 case Builtin::BI__builtin_bswap16: 678 case Builtin::BI__builtin_bswap32: 679 case Builtin::BI__builtin_bswap64: { 680 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bswap)); 681 } 682 case Builtin::BI__builtin_bitreverse8: 683 case Builtin::BI__builtin_bitreverse16: 684 case Builtin::BI__builtin_bitreverse32: 685 case Builtin::BI__builtin_bitreverse64: { 686 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bitreverse)); 687 } 688 case Builtin::BI__builtin_object_size: { 689 unsigned Type = 690 E->getArg(1)->EvaluateKnownConstInt(getContext()).getZExtValue(); 691 auto *ResType = cast<llvm::IntegerType>(ConvertType(E->getType())); 692 693 // We pass this builtin onto the optimizer so that it can figure out the 694 // object size in more complex cases. 695 return RValue::get(emitBuiltinObjectSize(E->getArg(0), Type, ResType)); 696 } 697 case Builtin::BI__builtin_prefetch: { 698 Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0)); 699 // FIXME: Technically these constants should of type 'int', yes? 700 RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) : 701 llvm::ConstantInt::get(Int32Ty, 0); 702 Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : 703 llvm::ConstantInt::get(Int32Ty, 3); 704 Value *Data = llvm::ConstantInt::get(Int32Ty, 1); 705 Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 706 return RValue::get(Builder.CreateCall(F, {Address, RW, Locality, Data})); 707 } 708 case Builtin::BI__builtin_readcyclecounter: { 709 Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter); 710 return RValue::get(Builder.CreateCall(F)); 711 } 712 case Builtin::BI__builtin___clear_cache: { 713 Value *Begin = EmitScalarExpr(E->getArg(0)); 714 Value *End = EmitScalarExpr(E->getArg(1)); 715 Value *F = CGM.getIntrinsic(Intrinsic::clear_cache); 716 return RValue::get(Builder.CreateCall(F, {Begin, End})); 717 } 718 case Builtin::BI__builtin_trap: 719 return RValue::get(EmitTrapCall(Intrinsic::trap)); 720 case Builtin::BI__debugbreak: 721 return RValue::get(EmitTrapCall(Intrinsic::debugtrap)); 722 case Builtin::BI__builtin_unreachable: { 723 if (SanOpts.has(SanitizerKind::Unreachable)) { 724 SanitizerScope SanScope(this); 725 EmitCheck(std::make_pair(static_cast<llvm::Value *>(Builder.getFalse()), 726 SanitizerKind::Unreachable), 727 "builtin_unreachable", EmitCheckSourceLocation(E->getExprLoc()), 728 None); 729 } else 730 Builder.CreateUnreachable(); 731 732 // We do need to preserve an insertion point. 733 EmitBlock(createBasicBlock("unreachable.cont")); 734 735 return RValue::get(nullptr); 736 } 737 738 case Builtin::BI__builtin_powi: 739 case Builtin::BI__builtin_powif: 740 case Builtin::BI__builtin_powil: { 741 Value *Base = EmitScalarExpr(E->getArg(0)); 742 Value *Exponent = EmitScalarExpr(E->getArg(1)); 743 llvm::Type *ArgType = Base->getType(); 744 Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType); 745 return RValue::get(Builder.CreateCall(F, {Base, Exponent})); 746 } 747 748 case Builtin::BI__builtin_isgreater: 749 case Builtin::BI__builtin_isgreaterequal: 750 case Builtin::BI__builtin_isless: 751 case Builtin::BI__builtin_islessequal: 752 case Builtin::BI__builtin_islessgreater: 753 case Builtin::BI__builtin_isunordered: { 754 // Ordered comparisons: we know the arguments to these are matching scalar 755 // floating point values. 756 Value *LHS = EmitScalarExpr(E->getArg(0)); 757 Value *RHS = EmitScalarExpr(E->getArg(1)); 758 759 switch (BuiltinID) { 760 default: llvm_unreachable("Unknown ordered comparison"); 761 case Builtin::BI__builtin_isgreater: 762 LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp"); 763 break; 764 case Builtin::BI__builtin_isgreaterequal: 765 LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp"); 766 break; 767 case Builtin::BI__builtin_isless: 768 LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp"); 769 break; 770 case Builtin::BI__builtin_islessequal: 771 LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp"); 772 break; 773 case Builtin::BI__builtin_islessgreater: 774 LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp"); 775 break; 776 case Builtin::BI__builtin_isunordered: 777 LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp"); 778 break; 779 } 780 // ZExt bool to int type. 781 return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType()))); 782 } 783 case Builtin::BI__builtin_isnan: { 784 Value *V = EmitScalarExpr(E->getArg(0)); 785 V = Builder.CreateFCmpUNO(V, V, "cmp"); 786 return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); 787 } 788 789 case Builtin::BI__builtin_isinf: 790 case Builtin::BI__builtin_isfinite: { 791 // isinf(x) --> fabs(x) == infinity 792 // isfinite(x) --> fabs(x) != infinity 793 // x != NaN via the ordered compare in either case. 794 Value *V = EmitScalarExpr(E->getArg(0)); 795 Value *Fabs = EmitFAbs(*this, V); 796 Constant *Infinity = ConstantFP::getInfinity(V->getType()); 797 CmpInst::Predicate Pred = (BuiltinID == Builtin::BI__builtin_isinf) 798 ? CmpInst::FCMP_OEQ 799 : CmpInst::FCMP_ONE; 800 Value *FCmp = Builder.CreateFCmp(Pred, Fabs, Infinity, "cmpinf"); 801 return RValue::get(Builder.CreateZExt(FCmp, ConvertType(E->getType()))); 802 } 803 804 case Builtin::BI__builtin_isinf_sign: { 805 // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0 806 Value *Arg = EmitScalarExpr(E->getArg(0)); 807 Value *AbsArg = EmitFAbs(*this, Arg); 808 Value *IsInf = Builder.CreateFCmpOEQ( 809 AbsArg, ConstantFP::getInfinity(Arg->getType()), "isinf"); 810 Value *IsNeg = EmitSignBit(*this, Arg); 811 812 llvm::Type *IntTy = ConvertType(E->getType()); 813 Value *Zero = Constant::getNullValue(IntTy); 814 Value *One = ConstantInt::get(IntTy, 1); 815 Value *NegativeOne = ConstantInt::get(IntTy, -1); 816 Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One); 817 Value *Result = Builder.CreateSelect(IsInf, SignResult, Zero); 818 return RValue::get(Result); 819 } 820 821 case Builtin::BI__builtin_isnormal: { 822 // isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min 823 Value *V = EmitScalarExpr(E->getArg(0)); 824 Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq"); 825 826 Value *Abs = EmitFAbs(*this, V); 827 Value *IsLessThanInf = 828 Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),"isinf"); 829 APFloat Smallest = APFloat::getSmallestNormalized( 830 getContext().getFloatTypeSemantics(E->getArg(0)->getType())); 831 Value *IsNormal = 832 Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest), 833 "isnormal"); 834 V = Builder.CreateAnd(Eq, IsLessThanInf, "and"); 835 V = Builder.CreateAnd(V, IsNormal, "and"); 836 return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); 837 } 838 839 case Builtin::BI__builtin_fpclassify: { 840 Value *V = EmitScalarExpr(E->getArg(5)); 841 llvm::Type *Ty = ConvertType(E->getArg(5)->getType()); 842 843 // Create Result 844 BasicBlock *Begin = Builder.GetInsertBlock(); 845 BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn); 846 Builder.SetInsertPoint(End); 847 PHINode *Result = 848 Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4, 849 "fpclassify_result"); 850 851 // if (V==0) return FP_ZERO 852 Builder.SetInsertPoint(Begin); 853 Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty), 854 "iszero"); 855 Value *ZeroLiteral = EmitScalarExpr(E->getArg(4)); 856 BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn); 857 Builder.CreateCondBr(IsZero, End, NotZero); 858 Result->addIncoming(ZeroLiteral, Begin); 859 860 // if (V != V) return FP_NAN 861 Builder.SetInsertPoint(NotZero); 862 Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp"); 863 Value *NanLiteral = EmitScalarExpr(E->getArg(0)); 864 BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn); 865 Builder.CreateCondBr(IsNan, End, NotNan); 866 Result->addIncoming(NanLiteral, NotZero); 867 868 // if (fabs(V) == infinity) return FP_INFINITY 869 Builder.SetInsertPoint(NotNan); 870 Value *VAbs = EmitFAbs(*this, V); 871 Value *IsInf = 872 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()), 873 "isinf"); 874 Value *InfLiteral = EmitScalarExpr(E->getArg(1)); 875 BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn); 876 Builder.CreateCondBr(IsInf, End, NotInf); 877 Result->addIncoming(InfLiteral, NotNan); 878 879 // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL 880 Builder.SetInsertPoint(NotInf); 881 APFloat Smallest = APFloat::getSmallestNormalized( 882 getContext().getFloatTypeSemantics(E->getArg(5)->getType())); 883 Value *IsNormal = 884 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest), 885 "isnormal"); 886 Value *NormalResult = 887 Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)), 888 EmitScalarExpr(E->getArg(3))); 889 Builder.CreateBr(End); 890 Result->addIncoming(NormalResult, NotInf); 891 892 // return Result 893 Builder.SetInsertPoint(End); 894 return RValue::get(Result); 895 } 896 897 case Builtin::BIalloca: 898 case Builtin::BI_alloca: 899 case Builtin::BI__builtin_alloca: { 900 Value *Size = EmitScalarExpr(E->getArg(0)); 901 return RValue::get(Builder.CreateAlloca(Builder.getInt8Ty(), Size)); 902 } 903 case Builtin::BIbzero: 904 case Builtin::BI__builtin_bzero: { 905 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 906 Value *SizeVal = EmitScalarExpr(E->getArg(1)); 907 EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), 908 E->getArg(0)->getExprLoc(), FD, 0); 909 Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false); 910 return RValue::get(Dest.getPointer()); 911 } 912 case Builtin::BImemcpy: 913 case Builtin::BI__builtin_memcpy: { 914 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 915 Address Src = EmitPointerWithAlignment(E->getArg(1)); 916 Value *SizeVal = EmitScalarExpr(E->getArg(2)); 917 EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), 918 E->getArg(0)->getExprLoc(), FD, 0); 919 EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(), 920 E->getArg(1)->getExprLoc(), FD, 1); 921 Builder.CreateMemCpy(Dest, Src, SizeVal, false); 922 return RValue::get(Dest.getPointer()); 923 } 924 925 case Builtin::BI__builtin___memcpy_chk: { 926 // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2. 927 llvm::APSInt Size, DstSize; 928 if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 929 !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 930 break; 931 if (Size.ugt(DstSize)) 932 break; 933 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 934 Address Src = EmitPointerWithAlignment(E->getArg(1)); 935 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 936 Builder.CreateMemCpy(Dest, Src, SizeVal, false); 937 return RValue::get(Dest.getPointer()); 938 } 939 940 case Builtin::BI__builtin_objc_memmove_collectable: { 941 Address DestAddr = EmitPointerWithAlignment(E->getArg(0)); 942 Address SrcAddr = EmitPointerWithAlignment(E->getArg(1)); 943 Value *SizeVal = EmitScalarExpr(E->getArg(2)); 944 CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, 945 DestAddr, SrcAddr, SizeVal); 946 return RValue::get(DestAddr.getPointer()); 947 } 948 949 case Builtin::BI__builtin___memmove_chk: { 950 // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2. 951 llvm::APSInt Size, DstSize; 952 if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 953 !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 954 break; 955 if (Size.ugt(DstSize)) 956 break; 957 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 958 Address Src = EmitPointerWithAlignment(E->getArg(1)); 959 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 960 Builder.CreateMemMove(Dest, Src, SizeVal, false); 961 return RValue::get(Dest.getPointer()); 962 } 963 964 case Builtin::BImemmove: 965 case Builtin::BI__builtin_memmove: { 966 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 967 Address Src = EmitPointerWithAlignment(E->getArg(1)); 968 Value *SizeVal = EmitScalarExpr(E->getArg(2)); 969 EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), 970 E->getArg(0)->getExprLoc(), FD, 0); 971 EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(), 972 E->getArg(1)->getExprLoc(), FD, 1); 973 Builder.CreateMemMove(Dest, Src, SizeVal, false); 974 return RValue::get(Dest.getPointer()); 975 } 976 case Builtin::BImemset: 977 case Builtin::BI__builtin_memset: { 978 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 979 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), 980 Builder.getInt8Ty()); 981 Value *SizeVal = EmitScalarExpr(E->getArg(2)); 982 EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), 983 E->getArg(0)->getExprLoc(), FD, 0); 984 Builder.CreateMemSet(Dest, ByteVal, SizeVal, false); 985 return RValue::get(Dest.getPointer()); 986 } 987 case Builtin::BI__builtin___memset_chk: { 988 // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2. 989 llvm::APSInt Size, DstSize; 990 if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 991 !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 992 break; 993 if (Size.ugt(DstSize)) 994 break; 995 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 996 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), 997 Builder.getInt8Ty()); 998 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 999 Builder.CreateMemSet(Dest, ByteVal, SizeVal, false); 1000 return RValue::get(Dest.getPointer()); 1001 } 1002 case Builtin::BI__builtin_dwarf_cfa: { 1003 // The offset in bytes from the first argument to the CFA. 1004 // 1005 // Why on earth is this in the frontend? Is there any reason at 1006 // all that the backend can't reasonably determine this while 1007 // lowering llvm.eh.dwarf.cfa()? 1008 // 1009 // TODO: If there's a satisfactory reason, add a target hook for 1010 // this instead of hard-coding 0, which is correct for most targets. 1011 int32_t Offset = 0; 1012 1013 Value *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa); 1014 return RValue::get(Builder.CreateCall(F, 1015 llvm::ConstantInt::get(Int32Ty, Offset))); 1016 } 1017 case Builtin::BI__builtin_return_address: { 1018 Value *Depth = 1019 CGM.EmitConstantExpr(E->getArg(0), getContext().UnsignedIntTy, this); 1020 Value *F = CGM.getIntrinsic(Intrinsic::returnaddress); 1021 return RValue::get(Builder.CreateCall(F, Depth)); 1022 } 1023 case Builtin::BI__builtin_frame_address: { 1024 Value *Depth = 1025 CGM.EmitConstantExpr(E->getArg(0), getContext().UnsignedIntTy, this); 1026 Value *F = CGM.getIntrinsic(Intrinsic::frameaddress); 1027 return RValue::get(Builder.CreateCall(F, Depth)); 1028 } 1029 case Builtin::BI__builtin_extract_return_addr: { 1030 Value *Address = EmitScalarExpr(E->getArg(0)); 1031 Value *Result = getTargetHooks().decodeReturnAddress(*this, Address); 1032 return RValue::get(Result); 1033 } 1034 case Builtin::BI__builtin_frob_return_addr: { 1035 Value *Address = EmitScalarExpr(E->getArg(0)); 1036 Value *Result = getTargetHooks().encodeReturnAddress(*this, Address); 1037 return RValue::get(Result); 1038 } 1039 case Builtin::BI__builtin_dwarf_sp_column: { 1040 llvm::IntegerType *Ty 1041 = cast<llvm::IntegerType>(ConvertType(E->getType())); 1042 int Column = getTargetHooks().getDwarfEHStackPointer(CGM); 1043 if (Column == -1) { 1044 CGM.ErrorUnsupported(E, "__builtin_dwarf_sp_column"); 1045 return RValue::get(llvm::UndefValue::get(Ty)); 1046 } 1047 return RValue::get(llvm::ConstantInt::get(Ty, Column, true)); 1048 } 1049 case Builtin::BI__builtin_init_dwarf_reg_size_table: { 1050 Value *Address = EmitScalarExpr(E->getArg(0)); 1051 if (getTargetHooks().initDwarfEHRegSizeTable(*this, Address)) 1052 CGM.ErrorUnsupported(E, "__builtin_init_dwarf_reg_size_table"); 1053 return RValue::get(llvm::UndefValue::get(ConvertType(E->getType()))); 1054 } 1055 case Builtin::BI__builtin_eh_return: { 1056 Value *Int = EmitScalarExpr(E->getArg(0)); 1057 Value *Ptr = EmitScalarExpr(E->getArg(1)); 1058 1059 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType()); 1060 assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) && 1061 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants"); 1062 Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32 1063 ? Intrinsic::eh_return_i32 1064 : Intrinsic::eh_return_i64); 1065 Builder.CreateCall(F, {Int, Ptr}); 1066 Builder.CreateUnreachable(); 1067 1068 // We do need to preserve an insertion point. 1069 EmitBlock(createBasicBlock("builtin_eh_return.cont")); 1070 1071 return RValue::get(nullptr); 1072 } 1073 case Builtin::BI__builtin_unwind_init: { 1074 Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init); 1075 return RValue::get(Builder.CreateCall(F)); 1076 } 1077 case Builtin::BI__builtin_extend_pointer: { 1078 // Extends a pointer to the size of an _Unwind_Word, which is 1079 // uint64_t on all platforms. Generally this gets poked into a 1080 // register and eventually used as an address, so if the 1081 // addressing registers are wider than pointers and the platform 1082 // doesn't implicitly ignore high-order bits when doing 1083 // addressing, we need to make sure we zext / sext based on 1084 // the platform's expectations. 1085 // 1086 // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html 1087 1088 // Cast the pointer to intptr_t. 1089 Value *Ptr = EmitScalarExpr(E->getArg(0)); 1090 Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast"); 1091 1092 // If that's 64 bits, we're done. 1093 if (IntPtrTy->getBitWidth() == 64) 1094 return RValue::get(Result); 1095 1096 // Otherwise, ask the codegen data what to do. 1097 if (getTargetHooks().extendPointerWithSExt()) 1098 return RValue::get(Builder.CreateSExt(Result, Int64Ty, "extend.sext")); 1099 else 1100 return RValue::get(Builder.CreateZExt(Result, Int64Ty, "extend.zext")); 1101 } 1102 case Builtin::BI__builtin_setjmp: { 1103 // Buffer is a void**. 1104 Address Buf = EmitPointerWithAlignment(E->getArg(0)); 1105 1106 // Store the frame pointer to the setjmp buffer. 1107 Value *FrameAddr = 1108 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), 1109 ConstantInt::get(Int32Ty, 0)); 1110 Builder.CreateStore(FrameAddr, Buf); 1111 1112 // Store the stack pointer to the setjmp buffer. 1113 Value *StackAddr = 1114 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave)); 1115 Address StackSaveSlot = 1116 Builder.CreateConstInBoundsGEP(Buf, 2, getPointerSize()); 1117 Builder.CreateStore(StackAddr, StackSaveSlot); 1118 1119 // Call LLVM's EH setjmp, which is lightweight. 1120 Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp); 1121 Buf = Builder.CreateBitCast(Buf, Int8PtrTy); 1122 return RValue::get(Builder.CreateCall(F, Buf.getPointer())); 1123 } 1124 case Builtin::BI__builtin_longjmp: { 1125 Value *Buf = EmitScalarExpr(E->getArg(0)); 1126 Buf = Builder.CreateBitCast(Buf, Int8PtrTy); 1127 1128 // Call LLVM's EH longjmp, which is lightweight. 1129 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf); 1130 1131 // longjmp doesn't return; mark this as unreachable. 1132 Builder.CreateUnreachable(); 1133 1134 // We do need to preserve an insertion point. 1135 EmitBlock(createBasicBlock("longjmp.cont")); 1136 1137 return RValue::get(nullptr); 1138 } 1139 case Builtin::BI__sync_fetch_and_add: 1140 case Builtin::BI__sync_fetch_and_sub: 1141 case Builtin::BI__sync_fetch_and_or: 1142 case Builtin::BI__sync_fetch_and_and: 1143 case Builtin::BI__sync_fetch_and_xor: 1144 case Builtin::BI__sync_fetch_and_nand: 1145 case Builtin::BI__sync_add_and_fetch: 1146 case Builtin::BI__sync_sub_and_fetch: 1147 case Builtin::BI__sync_and_and_fetch: 1148 case Builtin::BI__sync_or_and_fetch: 1149 case Builtin::BI__sync_xor_and_fetch: 1150 case Builtin::BI__sync_nand_and_fetch: 1151 case Builtin::BI__sync_val_compare_and_swap: 1152 case Builtin::BI__sync_bool_compare_and_swap: 1153 case Builtin::BI__sync_lock_test_and_set: 1154 case Builtin::BI__sync_lock_release: 1155 case Builtin::BI__sync_swap: 1156 llvm_unreachable("Shouldn't make it through sema"); 1157 case Builtin::BI__sync_fetch_and_add_1: 1158 case Builtin::BI__sync_fetch_and_add_2: 1159 case Builtin::BI__sync_fetch_and_add_4: 1160 case Builtin::BI__sync_fetch_and_add_8: 1161 case Builtin::BI__sync_fetch_and_add_16: 1162 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Add, E); 1163 case Builtin::BI__sync_fetch_and_sub_1: 1164 case Builtin::BI__sync_fetch_and_sub_2: 1165 case Builtin::BI__sync_fetch_and_sub_4: 1166 case Builtin::BI__sync_fetch_and_sub_8: 1167 case Builtin::BI__sync_fetch_and_sub_16: 1168 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Sub, E); 1169 case Builtin::BI__sync_fetch_and_or_1: 1170 case Builtin::BI__sync_fetch_and_or_2: 1171 case Builtin::BI__sync_fetch_and_or_4: 1172 case Builtin::BI__sync_fetch_and_or_8: 1173 case Builtin::BI__sync_fetch_and_or_16: 1174 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Or, E); 1175 case Builtin::BI__sync_fetch_and_and_1: 1176 case Builtin::BI__sync_fetch_and_and_2: 1177 case Builtin::BI__sync_fetch_and_and_4: 1178 case Builtin::BI__sync_fetch_and_and_8: 1179 case Builtin::BI__sync_fetch_and_and_16: 1180 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::And, E); 1181 case Builtin::BI__sync_fetch_and_xor_1: 1182 case Builtin::BI__sync_fetch_and_xor_2: 1183 case Builtin::BI__sync_fetch_and_xor_4: 1184 case Builtin::BI__sync_fetch_and_xor_8: 1185 case Builtin::BI__sync_fetch_and_xor_16: 1186 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xor, E); 1187 case Builtin::BI__sync_fetch_and_nand_1: 1188 case Builtin::BI__sync_fetch_and_nand_2: 1189 case Builtin::BI__sync_fetch_and_nand_4: 1190 case Builtin::BI__sync_fetch_and_nand_8: 1191 case Builtin::BI__sync_fetch_and_nand_16: 1192 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Nand, E); 1193 1194 // Clang extensions: not overloaded yet. 1195 case Builtin::BI__sync_fetch_and_min: 1196 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Min, E); 1197 case Builtin::BI__sync_fetch_and_max: 1198 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Max, E); 1199 case Builtin::BI__sync_fetch_and_umin: 1200 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMin, E); 1201 case Builtin::BI__sync_fetch_and_umax: 1202 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMax, E); 1203 1204 case Builtin::BI__sync_add_and_fetch_1: 1205 case Builtin::BI__sync_add_and_fetch_2: 1206 case Builtin::BI__sync_add_and_fetch_4: 1207 case Builtin::BI__sync_add_and_fetch_8: 1208 case Builtin::BI__sync_add_and_fetch_16: 1209 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Add, E, 1210 llvm::Instruction::Add); 1211 case Builtin::BI__sync_sub_and_fetch_1: 1212 case Builtin::BI__sync_sub_and_fetch_2: 1213 case Builtin::BI__sync_sub_and_fetch_4: 1214 case Builtin::BI__sync_sub_and_fetch_8: 1215 case Builtin::BI__sync_sub_and_fetch_16: 1216 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Sub, E, 1217 llvm::Instruction::Sub); 1218 case Builtin::BI__sync_and_and_fetch_1: 1219 case Builtin::BI__sync_and_and_fetch_2: 1220 case Builtin::BI__sync_and_and_fetch_4: 1221 case Builtin::BI__sync_and_and_fetch_8: 1222 case Builtin::BI__sync_and_and_fetch_16: 1223 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::And, E, 1224 llvm::Instruction::And); 1225 case Builtin::BI__sync_or_and_fetch_1: 1226 case Builtin::BI__sync_or_and_fetch_2: 1227 case Builtin::BI__sync_or_and_fetch_4: 1228 case Builtin::BI__sync_or_and_fetch_8: 1229 case Builtin::BI__sync_or_and_fetch_16: 1230 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Or, E, 1231 llvm::Instruction::Or); 1232 case Builtin::BI__sync_xor_and_fetch_1: 1233 case Builtin::BI__sync_xor_and_fetch_2: 1234 case Builtin::BI__sync_xor_and_fetch_4: 1235 case Builtin::BI__sync_xor_and_fetch_8: 1236 case Builtin::BI__sync_xor_and_fetch_16: 1237 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Xor, E, 1238 llvm::Instruction::Xor); 1239 case Builtin::BI__sync_nand_and_fetch_1: 1240 case Builtin::BI__sync_nand_and_fetch_2: 1241 case Builtin::BI__sync_nand_and_fetch_4: 1242 case Builtin::BI__sync_nand_and_fetch_8: 1243 case Builtin::BI__sync_nand_and_fetch_16: 1244 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Nand, E, 1245 llvm::Instruction::And, true); 1246 1247 case Builtin::BI__sync_val_compare_and_swap_1: 1248 case Builtin::BI__sync_val_compare_and_swap_2: 1249 case Builtin::BI__sync_val_compare_and_swap_4: 1250 case Builtin::BI__sync_val_compare_and_swap_8: 1251 case Builtin::BI__sync_val_compare_and_swap_16: 1252 return RValue::get(MakeAtomicCmpXchgValue(*this, E, false)); 1253 1254 case Builtin::BI__sync_bool_compare_and_swap_1: 1255 case Builtin::BI__sync_bool_compare_and_swap_2: 1256 case Builtin::BI__sync_bool_compare_and_swap_4: 1257 case Builtin::BI__sync_bool_compare_and_swap_8: 1258 case Builtin::BI__sync_bool_compare_and_swap_16: 1259 return RValue::get(MakeAtomicCmpXchgValue(*this, E, true)); 1260 1261 case Builtin::BI__sync_swap_1: 1262 case Builtin::BI__sync_swap_2: 1263 case Builtin::BI__sync_swap_4: 1264 case Builtin::BI__sync_swap_8: 1265 case Builtin::BI__sync_swap_16: 1266 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); 1267 1268 case Builtin::BI__sync_lock_test_and_set_1: 1269 case Builtin::BI__sync_lock_test_and_set_2: 1270 case Builtin::BI__sync_lock_test_and_set_4: 1271 case Builtin::BI__sync_lock_test_and_set_8: 1272 case Builtin::BI__sync_lock_test_and_set_16: 1273 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); 1274 1275 case Builtin::BI__sync_lock_release_1: 1276 case Builtin::BI__sync_lock_release_2: 1277 case Builtin::BI__sync_lock_release_4: 1278 case Builtin::BI__sync_lock_release_8: 1279 case Builtin::BI__sync_lock_release_16: { 1280 Value *Ptr = EmitScalarExpr(E->getArg(0)); 1281 QualType ElTy = E->getArg(0)->getType()->getPointeeType(); 1282 CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy); 1283 llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(), 1284 StoreSize.getQuantity() * 8); 1285 Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo()); 1286 llvm::StoreInst *Store = 1287 Builder.CreateAlignedStore(llvm::Constant::getNullValue(ITy), Ptr, 1288 StoreSize); 1289 Store->setAtomic(llvm::AtomicOrdering::Release); 1290 return RValue::get(nullptr); 1291 } 1292 1293 case Builtin::BI__sync_synchronize: { 1294 // We assume this is supposed to correspond to a C++0x-style 1295 // sequentially-consistent fence (i.e. this is only usable for 1296 // synchonization, not device I/O or anything like that). This intrinsic 1297 // is really badly designed in the sense that in theory, there isn't 1298 // any way to safely use it... but in practice, it mostly works 1299 // to use it with non-atomic loads and stores to get acquire/release 1300 // semantics. 1301 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent); 1302 return RValue::get(nullptr); 1303 } 1304 1305 case Builtin::BI__builtin_nontemporal_load: 1306 return RValue::get(EmitNontemporalLoad(*this, E)); 1307 case Builtin::BI__builtin_nontemporal_store: 1308 return RValue::get(EmitNontemporalStore(*this, E)); 1309 case Builtin::BI__c11_atomic_is_lock_free: 1310 case Builtin::BI__atomic_is_lock_free: { 1311 // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the 1312 // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since 1313 // _Atomic(T) is always properly-aligned. 1314 const char *LibCallName = "__atomic_is_lock_free"; 1315 CallArgList Args; 1316 Args.add(RValue::get(EmitScalarExpr(E->getArg(0))), 1317 getContext().getSizeType()); 1318 if (BuiltinID == Builtin::BI__atomic_is_lock_free) 1319 Args.add(RValue::get(EmitScalarExpr(E->getArg(1))), 1320 getContext().VoidPtrTy); 1321 else 1322 Args.add(RValue::get(llvm::Constant::getNullValue(VoidPtrTy)), 1323 getContext().VoidPtrTy); 1324 const CGFunctionInfo &FuncInfo = 1325 CGM.getTypes().arrangeBuiltinFunctionCall(E->getType(), Args); 1326 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo); 1327 llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName); 1328 return EmitCall(FuncInfo, Func, ReturnValueSlot(), Args); 1329 } 1330 1331 case Builtin::BI__atomic_test_and_set: { 1332 // Look at the argument type to determine whether this is a volatile 1333 // operation. The parameter type is always volatile. 1334 QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType(); 1335 bool Volatile = 1336 PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified(); 1337 1338 Value *Ptr = EmitScalarExpr(E->getArg(0)); 1339 unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace(); 1340 Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace)); 1341 Value *NewVal = Builder.getInt8(1); 1342 Value *Order = EmitScalarExpr(E->getArg(1)); 1343 if (isa<llvm::ConstantInt>(Order)) { 1344 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1345 AtomicRMWInst *Result = nullptr; 1346 switch (ord) { 1347 case 0: // memory_order_relaxed 1348 default: // invalid order 1349 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal, 1350 llvm::AtomicOrdering::Monotonic); 1351 break; 1352 case 1: // memory_order_consume 1353 case 2: // memory_order_acquire 1354 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal, 1355 llvm::AtomicOrdering::Acquire); 1356 break; 1357 case 3: // memory_order_release 1358 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal, 1359 llvm::AtomicOrdering::Release); 1360 break; 1361 case 4: // memory_order_acq_rel 1362 1363 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal, 1364 llvm::AtomicOrdering::AcquireRelease); 1365 break; 1366 case 5: // memory_order_seq_cst 1367 Result = Builder.CreateAtomicRMW( 1368 llvm::AtomicRMWInst::Xchg, Ptr, NewVal, 1369 llvm::AtomicOrdering::SequentiallyConsistent); 1370 break; 1371 } 1372 Result->setVolatile(Volatile); 1373 return RValue::get(Builder.CreateIsNotNull(Result, "tobool")); 1374 } 1375 1376 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1377 1378 llvm::BasicBlock *BBs[5] = { 1379 createBasicBlock("monotonic", CurFn), 1380 createBasicBlock("acquire", CurFn), 1381 createBasicBlock("release", CurFn), 1382 createBasicBlock("acqrel", CurFn), 1383 createBasicBlock("seqcst", CurFn) 1384 }; 1385 llvm::AtomicOrdering Orders[5] = { 1386 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire, 1387 llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease, 1388 llvm::AtomicOrdering::SequentiallyConsistent}; 1389 1390 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1391 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]); 1392 1393 Builder.SetInsertPoint(ContBB); 1394 PHINode *Result = Builder.CreatePHI(Int8Ty, 5, "was_set"); 1395 1396 for (unsigned i = 0; i < 5; ++i) { 1397 Builder.SetInsertPoint(BBs[i]); 1398 AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 1399 Ptr, NewVal, Orders[i]); 1400 RMW->setVolatile(Volatile); 1401 Result->addIncoming(RMW, BBs[i]); 1402 Builder.CreateBr(ContBB); 1403 } 1404 1405 SI->addCase(Builder.getInt32(0), BBs[0]); 1406 SI->addCase(Builder.getInt32(1), BBs[1]); 1407 SI->addCase(Builder.getInt32(2), BBs[1]); 1408 SI->addCase(Builder.getInt32(3), BBs[2]); 1409 SI->addCase(Builder.getInt32(4), BBs[3]); 1410 SI->addCase(Builder.getInt32(5), BBs[4]); 1411 1412 Builder.SetInsertPoint(ContBB); 1413 return RValue::get(Builder.CreateIsNotNull(Result, "tobool")); 1414 } 1415 1416 case Builtin::BI__atomic_clear: { 1417 QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType(); 1418 bool Volatile = 1419 PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified(); 1420 1421 Address Ptr = EmitPointerWithAlignment(E->getArg(0)); 1422 unsigned AddrSpace = Ptr.getPointer()->getType()->getPointerAddressSpace(); 1423 Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace)); 1424 Value *NewVal = Builder.getInt8(0); 1425 Value *Order = EmitScalarExpr(E->getArg(1)); 1426 if (isa<llvm::ConstantInt>(Order)) { 1427 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1428 StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile); 1429 switch (ord) { 1430 case 0: // memory_order_relaxed 1431 default: // invalid order 1432 Store->setOrdering(llvm::AtomicOrdering::Monotonic); 1433 break; 1434 case 3: // memory_order_release 1435 Store->setOrdering(llvm::AtomicOrdering::Release); 1436 break; 1437 case 5: // memory_order_seq_cst 1438 Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent); 1439 break; 1440 } 1441 return RValue::get(nullptr); 1442 } 1443 1444 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1445 1446 llvm::BasicBlock *BBs[3] = { 1447 createBasicBlock("monotonic", CurFn), 1448 createBasicBlock("release", CurFn), 1449 createBasicBlock("seqcst", CurFn) 1450 }; 1451 llvm::AtomicOrdering Orders[3] = { 1452 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release, 1453 llvm::AtomicOrdering::SequentiallyConsistent}; 1454 1455 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1456 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]); 1457 1458 for (unsigned i = 0; i < 3; ++i) { 1459 Builder.SetInsertPoint(BBs[i]); 1460 StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile); 1461 Store->setOrdering(Orders[i]); 1462 Builder.CreateBr(ContBB); 1463 } 1464 1465 SI->addCase(Builder.getInt32(0), BBs[0]); 1466 SI->addCase(Builder.getInt32(3), BBs[1]); 1467 SI->addCase(Builder.getInt32(5), BBs[2]); 1468 1469 Builder.SetInsertPoint(ContBB); 1470 return RValue::get(nullptr); 1471 } 1472 1473 case Builtin::BI__atomic_thread_fence: 1474 case Builtin::BI__atomic_signal_fence: 1475 case Builtin::BI__c11_atomic_thread_fence: 1476 case Builtin::BI__c11_atomic_signal_fence: { 1477 llvm::SynchronizationScope Scope; 1478 if (BuiltinID == Builtin::BI__atomic_signal_fence || 1479 BuiltinID == Builtin::BI__c11_atomic_signal_fence) 1480 Scope = llvm::SingleThread; 1481 else 1482 Scope = llvm::CrossThread; 1483 Value *Order = EmitScalarExpr(E->getArg(0)); 1484 if (isa<llvm::ConstantInt>(Order)) { 1485 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1486 switch (ord) { 1487 case 0: // memory_order_relaxed 1488 default: // invalid order 1489 break; 1490 case 1: // memory_order_consume 1491 case 2: // memory_order_acquire 1492 Builder.CreateFence(llvm::AtomicOrdering::Acquire, Scope); 1493 break; 1494 case 3: // memory_order_release 1495 Builder.CreateFence(llvm::AtomicOrdering::Release, Scope); 1496 break; 1497 case 4: // memory_order_acq_rel 1498 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, Scope); 1499 break; 1500 case 5: // memory_order_seq_cst 1501 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, 1502 Scope); 1503 break; 1504 } 1505 return RValue::get(nullptr); 1506 } 1507 1508 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB; 1509 AcquireBB = createBasicBlock("acquire", CurFn); 1510 ReleaseBB = createBasicBlock("release", CurFn); 1511 AcqRelBB = createBasicBlock("acqrel", CurFn); 1512 SeqCstBB = createBasicBlock("seqcst", CurFn); 1513 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1514 1515 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1516 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB); 1517 1518 Builder.SetInsertPoint(AcquireBB); 1519 Builder.CreateFence(llvm::AtomicOrdering::Acquire, Scope); 1520 Builder.CreateBr(ContBB); 1521 SI->addCase(Builder.getInt32(1), AcquireBB); 1522 SI->addCase(Builder.getInt32(2), AcquireBB); 1523 1524 Builder.SetInsertPoint(ReleaseBB); 1525 Builder.CreateFence(llvm::AtomicOrdering::Release, Scope); 1526 Builder.CreateBr(ContBB); 1527 SI->addCase(Builder.getInt32(3), ReleaseBB); 1528 1529 Builder.SetInsertPoint(AcqRelBB); 1530 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, Scope); 1531 Builder.CreateBr(ContBB); 1532 SI->addCase(Builder.getInt32(4), AcqRelBB); 1533 1534 Builder.SetInsertPoint(SeqCstBB); 1535 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1536 Builder.CreateBr(ContBB); 1537 SI->addCase(Builder.getInt32(5), SeqCstBB); 1538 1539 Builder.SetInsertPoint(ContBB); 1540 return RValue::get(nullptr); 1541 } 1542 1543 // Library functions with special handling. 1544 case Builtin::BIsqrt: 1545 case Builtin::BIsqrtf: 1546 case Builtin::BIsqrtl: { 1547 // Transform a call to sqrt* into a @llvm.sqrt.* intrinsic call, but only 1548 // in finite- or unsafe-math mode (the intrinsic has different semantics 1549 // for handling negative numbers compared to the library function, so 1550 // -fmath-errno=0 is not enough). 1551 if (!FD->hasAttr<ConstAttr>()) 1552 break; 1553 if (!(CGM.getCodeGenOpts().UnsafeFPMath || 1554 CGM.getCodeGenOpts().NoNaNsFPMath)) 1555 break; 1556 Value *Arg0 = EmitScalarExpr(E->getArg(0)); 1557 llvm::Type *ArgType = Arg0->getType(); 1558 Value *F = CGM.getIntrinsic(Intrinsic::sqrt, ArgType); 1559 return RValue::get(Builder.CreateCall(F, Arg0)); 1560 } 1561 1562 case Builtin::BI__builtin_pow: 1563 case Builtin::BI__builtin_powf: 1564 case Builtin::BI__builtin_powl: 1565 case Builtin::BIpow: 1566 case Builtin::BIpowf: 1567 case Builtin::BIpowl: { 1568 // Transform a call to pow* into a @llvm.pow.* intrinsic call. 1569 if (!FD->hasAttr<ConstAttr>()) 1570 break; 1571 Value *Base = EmitScalarExpr(E->getArg(0)); 1572 Value *Exponent = EmitScalarExpr(E->getArg(1)); 1573 llvm::Type *ArgType = Base->getType(); 1574 Value *F = CGM.getIntrinsic(Intrinsic::pow, ArgType); 1575 return RValue::get(Builder.CreateCall(F, {Base, Exponent})); 1576 } 1577 1578 case Builtin::BIfma: 1579 case Builtin::BIfmaf: 1580 case Builtin::BIfmal: 1581 case Builtin::BI__builtin_fma: 1582 case Builtin::BI__builtin_fmaf: 1583 case Builtin::BI__builtin_fmal: { 1584 // Rewrite fma to intrinsic. 1585 Value *FirstArg = EmitScalarExpr(E->getArg(0)); 1586 llvm::Type *ArgType = FirstArg->getType(); 1587 Value *F = CGM.getIntrinsic(Intrinsic::fma, ArgType); 1588 return RValue::get( 1589 Builder.CreateCall(F, {FirstArg, EmitScalarExpr(E->getArg(1)), 1590 EmitScalarExpr(E->getArg(2))})); 1591 } 1592 1593 case Builtin::BI__builtin_signbit: 1594 case Builtin::BI__builtin_signbitf: 1595 case Builtin::BI__builtin_signbitl: { 1596 return RValue::get( 1597 Builder.CreateZExt(EmitSignBit(*this, EmitScalarExpr(E->getArg(0))), 1598 ConvertType(E->getType()))); 1599 } 1600 case Builtin::BI__builtin_annotation: { 1601 llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0)); 1602 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::annotation, 1603 AnnVal->getType()); 1604 1605 // Get the annotation string, go through casts. Sema requires this to be a 1606 // non-wide string literal, potentially casted, so the cast<> is safe. 1607 const Expr *AnnotationStrExpr = E->getArg(1)->IgnoreParenCasts(); 1608 StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString(); 1609 return RValue::get(EmitAnnotationCall(F, AnnVal, Str, E->getExprLoc())); 1610 } 1611 case Builtin::BI__builtin_addcb: 1612 case Builtin::BI__builtin_addcs: 1613 case Builtin::BI__builtin_addc: 1614 case Builtin::BI__builtin_addcl: 1615 case Builtin::BI__builtin_addcll: 1616 case Builtin::BI__builtin_subcb: 1617 case Builtin::BI__builtin_subcs: 1618 case Builtin::BI__builtin_subc: 1619 case Builtin::BI__builtin_subcl: 1620 case Builtin::BI__builtin_subcll: { 1621 1622 // We translate all of these builtins from expressions of the form: 1623 // int x = ..., y = ..., carryin = ..., carryout, result; 1624 // result = __builtin_addc(x, y, carryin, &carryout); 1625 // 1626 // to LLVM IR of the form: 1627 // 1628 // %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y) 1629 // %tmpsum1 = extractvalue {i32, i1} %tmp1, 0 1630 // %carry1 = extractvalue {i32, i1} %tmp1, 1 1631 // %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1, 1632 // i32 %carryin) 1633 // %result = extractvalue {i32, i1} %tmp2, 0 1634 // %carry2 = extractvalue {i32, i1} %tmp2, 1 1635 // %tmp3 = or i1 %carry1, %carry2 1636 // %tmp4 = zext i1 %tmp3 to i32 1637 // store i32 %tmp4, i32* %carryout 1638 1639 // Scalarize our inputs. 1640 llvm::Value *X = EmitScalarExpr(E->getArg(0)); 1641 llvm::Value *Y = EmitScalarExpr(E->getArg(1)); 1642 llvm::Value *Carryin = EmitScalarExpr(E->getArg(2)); 1643 Address CarryOutPtr = EmitPointerWithAlignment(E->getArg(3)); 1644 1645 // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow. 1646 llvm::Intrinsic::ID IntrinsicId; 1647 switch (BuiltinID) { 1648 default: llvm_unreachable("Unknown multiprecision builtin id."); 1649 case Builtin::BI__builtin_addcb: 1650 case Builtin::BI__builtin_addcs: 1651 case Builtin::BI__builtin_addc: 1652 case Builtin::BI__builtin_addcl: 1653 case Builtin::BI__builtin_addcll: 1654 IntrinsicId = llvm::Intrinsic::uadd_with_overflow; 1655 break; 1656 case Builtin::BI__builtin_subcb: 1657 case Builtin::BI__builtin_subcs: 1658 case Builtin::BI__builtin_subc: 1659 case Builtin::BI__builtin_subcl: 1660 case Builtin::BI__builtin_subcll: 1661 IntrinsicId = llvm::Intrinsic::usub_with_overflow; 1662 break; 1663 } 1664 1665 // Construct our resulting LLVM IR expression. 1666 llvm::Value *Carry1; 1667 llvm::Value *Sum1 = EmitOverflowIntrinsic(*this, IntrinsicId, 1668 X, Y, Carry1); 1669 llvm::Value *Carry2; 1670 llvm::Value *Sum2 = EmitOverflowIntrinsic(*this, IntrinsicId, 1671 Sum1, Carryin, Carry2); 1672 llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2), 1673 X->getType()); 1674 Builder.CreateStore(CarryOut, CarryOutPtr); 1675 return RValue::get(Sum2); 1676 } 1677 1678 case Builtin::BI__builtin_add_overflow: 1679 case Builtin::BI__builtin_sub_overflow: 1680 case Builtin::BI__builtin_mul_overflow: { 1681 const clang::Expr *LeftArg = E->getArg(0); 1682 const clang::Expr *RightArg = E->getArg(1); 1683 const clang::Expr *ResultArg = E->getArg(2); 1684 1685 clang::QualType ResultQTy = 1686 ResultArg->getType()->castAs<PointerType>()->getPointeeType(); 1687 1688 WidthAndSignedness LeftInfo = 1689 getIntegerWidthAndSignedness(CGM.getContext(), LeftArg->getType()); 1690 WidthAndSignedness RightInfo = 1691 getIntegerWidthAndSignedness(CGM.getContext(), RightArg->getType()); 1692 WidthAndSignedness ResultInfo = 1693 getIntegerWidthAndSignedness(CGM.getContext(), ResultQTy); 1694 WidthAndSignedness EncompassingInfo = 1695 EncompassingIntegerType({LeftInfo, RightInfo, ResultInfo}); 1696 1697 llvm::Type *EncompassingLLVMTy = 1698 llvm::IntegerType::get(CGM.getLLVMContext(), EncompassingInfo.Width); 1699 1700 llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(ResultQTy); 1701 1702 llvm::Intrinsic::ID IntrinsicId; 1703 switch (BuiltinID) { 1704 default: 1705 llvm_unreachable("Unknown overflow builtin id."); 1706 case Builtin::BI__builtin_add_overflow: 1707 IntrinsicId = EncompassingInfo.Signed 1708 ? llvm::Intrinsic::sadd_with_overflow 1709 : llvm::Intrinsic::uadd_with_overflow; 1710 break; 1711 case Builtin::BI__builtin_sub_overflow: 1712 IntrinsicId = EncompassingInfo.Signed 1713 ? llvm::Intrinsic::ssub_with_overflow 1714 : llvm::Intrinsic::usub_with_overflow; 1715 break; 1716 case Builtin::BI__builtin_mul_overflow: 1717 IntrinsicId = EncompassingInfo.Signed 1718 ? llvm::Intrinsic::smul_with_overflow 1719 : llvm::Intrinsic::umul_with_overflow; 1720 break; 1721 } 1722 1723 llvm::Value *Left = EmitScalarExpr(LeftArg); 1724 llvm::Value *Right = EmitScalarExpr(RightArg); 1725 Address ResultPtr = EmitPointerWithAlignment(ResultArg); 1726 1727 // Extend each operand to the encompassing type. 1728 Left = Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed); 1729 Right = Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed); 1730 1731 // Perform the operation on the extended values. 1732 llvm::Value *Overflow, *Result; 1733 Result = EmitOverflowIntrinsic(*this, IntrinsicId, Left, Right, Overflow); 1734 1735 if (EncompassingInfo.Width > ResultInfo.Width) { 1736 // The encompassing type is wider than the result type, so we need to 1737 // truncate it. 1738 llvm::Value *ResultTrunc = Builder.CreateTrunc(Result, ResultLLVMTy); 1739 1740 // To see if the truncation caused an overflow, we will extend 1741 // the result and then compare it to the original result. 1742 llvm::Value *ResultTruncExt = Builder.CreateIntCast( 1743 ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed); 1744 llvm::Value *TruncationOverflow = 1745 Builder.CreateICmpNE(Result, ResultTruncExt); 1746 1747 Overflow = Builder.CreateOr(Overflow, TruncationOverflow); 1748 Result = ResultTrunc; 1749 } 1750 1751 // Finally, store the result using the pointer. 1752 bool isVolatile = 1753 ResultArg->getType()->getPointeeType().isVolatileQualified(); 1754 Builder.CreateStore(EmitToMemory(Result, ResultQTy), ResultPtr, isVolatile); 1755 1756 return RValue::get(Overflow); 1757 } 1758 1759 case Builtin::BI__builtin_uadd_overflow: 1760 case Builtin::BI__builtin_uaddl_overflow: 1761 case Builtin::BI__builtin_uaddll_overflow: 1762 case Builtin::BI__builtin_usub_overflow: 1763 case Builtin::BI__builtin_usubl_overflow: 1764 case Builtin::BI__builtin_usubll_overflow: 1765 case Builtin::BI__builtin_umul_overflow: 1766 case Builtin::BI__builtin_umull_overflow: 1767 case Builtin::BI__builtin_umulll_overflow: 1768 case Builtin::BI__builtin_sadd_overflow: 1769 case Builtin::BI__builtin_saddl_overflow: 1770 case Builtin::BI__builtin_saddll_overflow: 1771 case Builtin::BI__builtin_ssub_overflow: 1772 case Builtin::BI__builtin_ssubl_overflow: 1773 case Builtin::BI__builtin_ssubll_overflow: 1774 case Builtin::BI__builtin_smul_overflow: 1775 case Builtin::BI__builtin_smull_overflow: 1776 case Builtin::BI__builtin_smulll_overflow: { 1777 1778 // We translate all of these builtins directly to the relevant llvm IR node. 1779 1780 // Scalarize our inputs. 1781 llvm::Value *X = EmitScalarExpr(E->getArg(0)); 1782 llvm::Value *Y = EmitScalarExpr(E->getArg(1)); 1783 Address SumOutPtr = EmitPointerWithAlignment(E->getArg(2)); 1784 1785 // Decide which of the overflow intrinsics we are lowering to: 1786 llvm::Intrinsic::ID IntrinsicId; 1787 switch (BuiltinID) { 1788 default: llvm_unreachable("Unknown overflow builtin id."); 1789 case Builtin::BI__builtin_uadd_overflow: 1790 case Builtin::BI__builtin_uaddl_overflow: 1791 case Builtin::BI__builtin_uaddll_overflow: 1792 IntrinsicId = llvm::Intrinsic::uadd_with_overflow; 1793 break; 1794 case Builtin::BI__builtin_usub_overflow: 1795 case Builtin::BI__builtin_usubl_overflow: 1796 case Builtin::BI__builtin_usubll_overflow: 1797 IntrinsicId = llvm::Intrinsic::usub_with_overflow; 1798 break; 1799 case Builtin::BI__builtin_umul_overflow: 1800 case Builtin::BI__builtin_umull_overflow: 1801 case Builtin::BI__builtin_umulll_overflow: 1802 IntrinsicId = llvm::Intrinsic::umul_with_overflow; 1803 break; 1804 case Builtin::BI__builtin_sadd_overflow: 1805 case Builtin::BI__builtin_saddl_overflow: 1806 case Builtin::BI__builtin_saddll_overflow: 1807 IntrinsicId = llvm::Intrinsic::sadd_with_overflow; 1808 break; 1809 case Builtin::BI__builtin_ssub_overflow: 1810 case Builtin::BI__builtin_ssubl_overflow: 1811 case Builtin::BI__builtin_ssubll_overflow: 1812 IntrinsicId = llvm::Intrinsic::ssub_with_overflow; 1813 break; 1814 case Builtin::BI__builtin_smul_overflow: 1815 case Builtin::BI__builtin_smull_overflow: 1816 case Builtin::BI__builtin_smulll_overflow: 1817 IntrinsicId = llvm::Intrinsic::smul_with_overflow; 1818 break; 1819 } 1820 1821 1822 llvm::Value *Carry; 1823 llvm::Value *Sum = EmitOverflowIntrinsic(*this, IntrinsicId, X, Y, Carry); 1824 Builder.CreateStore(Sum, SumOutPtr); 1825 1826 return RValue::get(Carry); 1827 } 1828 case Builtin::BI__builtin_addressof: 1829 return RValue::get(EmitLValue(E->getArg(0)).getPointer()); 1830 case Builtin::BI__builtin_operator_new: 1831 return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(), 1832 E->getArg(0), false); 1833 case Builtin::BI__builtin_operator_delete: 1834 return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(), 1835 E->getArg(0), true); 1836 case Builtin::BI__noop: 1837 // __noop always evaluates to an integer literal zero. 1838 return RValue::get(ConstantInt::get(IntTy, 0)); 1839 case Builtin::BI__builtin_call_with_static_chain: { 1840 const CallExpr *Call = cast<CallExpr>(E->getArg(0)); 1841 const Expr *Chain = E->getArg(1); 1842 return EmitCall(Call->getCallee()->getType(), 1843 EmitScalarExpr(Call->getCallee()), Call, ReturnValue, 1844 Call->getCalleeDecl(), EmitScalarExpr(Chain)); 1845 } 1846 case Builtin::BI_InterlockedExchange: 1847 case Builtin::BI_InterlockedExchangePointer: 1848 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); 1849 case Builtin::BI_InterlockedCompareExchangePointer: { 1850 llvm::Type *RTy; 1851 llvm::IntegerType *IntType = 1852 IntegerType::get(getLLVMContext(), 1853 getContext().getTypeSize(E->getType())); 1854 llvm::Type *IntPtrType = IntType->getPointerTo(); 1855 1856 llvm::Value *Destination = 1857 Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), IntPtrType); 1858 1859 llvm::Value *Exchange = EmitScalarExpr(E->getArg(1)); 1860 RTy = Exchange->getType(); 1861 Exchange = Builder.CreatePtrToInt(Exchange, IntType); 1862 1863 llvm::Value *Comparand = 1864 Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType); 1865 1866 auto Result = 1867 Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange, 1868 AtomicOrdering::SequentiallyConsistent, 1869 AtomicOrdering::SequentiallyConsistent); 1870 Result->setVolatile(true); 1871 1872 return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result, 1873 0), 1874 RTy)); 1875 } 1876 case Builtin::BI_InterlockedCompareExchange: { 1877 AtomicCmpXchgInst *CXI = Builder.CreateAtomicCmpXchg( 1878 EmitScalarExpr(E->getArg(0)), 1879 EmitScalarExpr(E->getArg(2)), 1880 EmitScalarExpr(E->getArg(1)), 1881 AtomicOrdering::SequentiallyConsistent, 1882 AtomicOrdering::SequentiallyConsistent); 1883 CXI->setVolatile(true); 1884 return RValue::get(Builder.CreateExtractValue(CXI, 0)); 1885 } 1886 case Builtin::BI_InterlockedIncrement: { 1887 llvm::Type *IntTy = ConvertType(E->getType()); 1888 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1889 AtomicRMWInst::Add, 1890 EmitScalarExpr(E->getArg(0)), 1891 ConstantInt::get(IntTy, 1), 1892 llvm::AtomicOrdering::SequentiallyConsistent); 1893 RMWI->setVolatile(true); 1894 return RValue::get(Builder.CreateAdd(RMWI, ConstantInt::get(IntTy, 1))); 1895 } 1896 case Builtin::BI_InterlockedDecrement: { 1897 llvm::Type *IntTy = ConvertType(E->getType()); 1898 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1899 AtomicRMWInst::Sub, 1900 EmitScalarExpr(E->getArg(0)), 1901 ConstantInt::get(IntTy, 1), 1902 llvm::AtomicOrdering::SequentiallyConsistent); 1903 RMWI->setVolatile(true); 1904 return RValue::get(Builder.CreateSub(RMWI, ConstantInt::get(IntTy, 1))); 1905 } 1906 case Builtin::BI_InterlockedExchangeAdd: { 1907 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1908 AtomicRMWInst::Add, 1909 EmitScalarExpr(E->getArg(0)), 1910 EmitScalarExpr(E->getArg(1)), 1911 llvm::AtomicOrdering::SequentiallyConsistent); 1912 RMWI->setVolatile(true); 1913 return RValue::get(RMWI); 1914 } 1915 case Builtin::BI__readfsdword: { 1916 llvm::Type *IntTy = ConvertType(E->getType()); 1917 Value *IntToPtr = 1918 Builder.CreateIntToPtr(EmitScalarExpr(E->getArg(0)), 1919 llvm::PointerType::get(IntTy, 257)); 1920 LoadInst *Load = 1921 Builder.CreateDefaultAlignedLoad(IntToPtr, /*isVolatile=*/true); 1922 return RValue::get(Load); 1923 } 1924 1925 case Builtin::BI__exception_code: 1926 case Builtin::BI_exception_code: 1927 return RValue::get(EmitSEHExceptionCode()); 1928 case Builtin::BI__exception_info: 1929 case Builtin::BI_exception_info: 1930 return RValue::get(EmitSEHExceptionInfo()); 1931 case Builtin::BI__abnormal_termination: 1932 case Builtin::BI_abnormal_termination: 1933 return RValue::get(EmitSEHAbnormalTermination()); 1934 case Builtin::BI_setjmpex: { 1935 if (getTarget().getTriple().isOSMSVCRT()) { 1936 llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy}; 1937 llvm::AttributeSet ReturnsTwiceAttr = 1938 AttributeSet::get(getLLVMContext(), llvm::AttributeSet::FunctionIndex, 1939 llvm::Attribute::ReturnsTwice); 1940 llvm::Constant *SetJmpEx = CGM.CreateRuntimeFunction( 1941 llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false), 1942 "_setjmpex", ReturnsTwiceAttr); 1943 llvm::Value *Buf = Builder.CreateBitOrPointerCast( 1944 EmitScalarExpr(E->getArg(0)), Int8PtrTy); 1945 llvm::Value *FrameAddr = 1946 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), 1947 ConstantInt::get(Int32Ty, 0)); 1948 llvm::Value *Args[] = {Buf, FrameAddr}; 1949 llvm::CallSite CS = EmitRuntimeCallOrInvoke(SetJmpEx, Args); 1950 CS.setAttributes(ReturnsTwiceAttr); 1951 return RValue::get(CS.getInstruction()); 1952 } 1953 break; 1954 } 1955 case Builtin::BI_setjmp: { 1956 if (getTarget().getTriple().isOSMSVCRT()) { 1957 llvm::AttributeSet ReturnsTwiceAttr = 1958 AttributeSet::get(getLLVMContext(), llvm::AttributeSet::FunctionIndex, 1959 llvm::Attribute::ReturnsTwice); 1960 llvm::Value *Buf = Builder.CreateBitOrPointerCast( 1961 EmitScalarExpr(E->getArg(0)), Int8PtrTy); 1962 llvm::CallSite CS; 1963 if (getTarget().getTriple().getArch() == llvm::Triple::x86) { 1964 llvm::Type *ArgTypes[] = {Int8PtrTy, IntTy}; 1965 llvm::Constant *SetJmp3 = CGM.CreateRuntimeFunction( 1966 llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/true), 1967 "_setjmp3", ReturnsTwiceAttr); 1968 llvm::Value *Count = ConstantInt::get(IntTy, 0); 1969 llvm::Value *Args[] = {Buf, Count}; 1970 CS = EmitRuntimeCallOrInvoke(SetJmp3, Args); 1971 } else { 1972 llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy}; 1973 llvm::Constant *SetJmp = CGM.CreateRuntimeFunction( 1974 llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false), 1975 "_setjmp", ReturnsTwiceAttr); 1976 llvm::Value *FrameAddr = 1977 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), 1978 ConstantInt::get(Int32Ty, 0)); 1979 llvm::Value *Args[] = {Buf, FrameAddr}; 1980 CS = EmitRuntimeCallOrInvoke(SetJmp, Args); 1981 } 1982 CS.setAttributes(ReturnsTwiceAttr); 1983 return RValue::get(CS.getInstruction()); 1984 } 1985 break; 1986 } 1987 1988 case Builtin::BI__GetExceptionInfo: { 1989 if (llvm::GlobalVariable *GV = 1990 CGM.getCXXABI().getThrowInfo(FD->getParamDecl(0)->getType())) 1991 return RValue::get(llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy)); 1992 break; 1993 } 1994 1995 // OpenCL v2.0 s6.13.16.2, Built-in pipe read and write functions 1996 case Builtin::BIread_pipe: 1997 case Builtin::BIwrite_pipe: { 1998 Value *Arg0 = EmitScalarExpr(E->getArg(0)), 1999 *Arg1 = EmitScalarExpr(E->getArg(1)); 2000 2001 // Type of the generic packet parameter. 2002 unsigned GenericAS = 2003 getContext().getTargetAddressSpace(LangAS::opencl_generic); 2004 llvm::Type *I8PTy = llvm::PointerType::get( 2005 llvm::Type::getInt8Ty(getLLVMContext()), GenericAS); 2006 2007 // Testing which overloaded version we should generate the call for. 2008 if (2U == E->getNumArgs()) { 2009 const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_2" 2010 : "__write_pipe_2"; 2011 // Creating a generic function type to be able to call with any builtin or 2012 // user defined type. 2013 llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy}; 2014 llvm::FunctionType *FTy = llvm::FunctionType::get( 2015 Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false); 2016 Value *BCast = Builder.CreatePointerCast(Arg1, I8PTy); 2017 return RValue::get(Builder.CreateCall( 2018 CGM.CreateRuntimeFunction(FTy, Name), {Arg0, BCast})); 2019 } else { 2020 assert(4 == E->getNumArgs() && 2021 "Illegal number of parameters to pipe function"); 2022 const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_4" 2023 : "__write_pipe_4"; 2024 2025 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy}; 2026 Value *Arg2 = EmitScalarExpr(E->getArg(2)), 2027 *Arg3 = EmitScalarExpr(E->getArg(3)); 2028 llvm::FunctionType *FTy = llvm::FunctionType::get( 2029 Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false); 2030 Value *BCast = Builder.CreatePointerCast(Arg3, I8PTy); 2031 // We know the third argument is an integer type, but we may need to cast 2032 // it to i32. 2033 if (Arg2->getType() != Int32Ty) 2034 Arg2 = Builder.CreateZExtOrTrunc(Arg2, Int32Ty); 2035 return RValue::get(Builder.CreateCall( 2036 CGM.CreateRuntimeFunction(FTy, Name), {Arg0, Arg1, Arg2, BCast})); 2037 } 2038 } 2039 // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe reserve read and write 2040 // functions 2041 case Builtin::BIreserve_read_pipe: 2042 case Builtin::BIreserve_write_pipe: 2043 case Builtin::BIwork_group_reserve_read_pipe: 2044 case Builtin::BIwork_group_reserve_write_pipe: 2045 case Builtin::BIsub_group_reserve_read_pipe: 2046 case Builtin::BIsub_group_reserve_write_pipe: { 2047 // Composing the mangled name for the function. 2048 const char *Name; 2049 if (BuiltinID == Builtin::BIreserve_read_pipe) 2050 Name = "__reserve_read_pipe"; 2051 else if (BuiltinID == Builtin::BIreserve_write_pipe) 2052 Name = "__reserve_write_pipe"; 2053 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe) 2054 Name = "__work_group_reserve_read_pipe"; 2055 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe) 2056 Name = "__work_group_reserve_write_pipe"; 2057 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe) 2058 Name = "__sub_group_reserve_read_pipe"; 2059 else 2060 Name = "__sub_group_reserve_write_pipe"; 2061 2062 Value *Arg0 = EmitScalarExpr(E->getArg(0)), 2063 *Arg1 = EmitScalarExpr(E->getArg(1)); 2064 llvm::Type *ReservedIDTy = ConvertType(getContext().OCLReserveIDTy); 2065 2066 // Building the generic function prototype. 2067 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty}; 2068 llvm::FunctionType *FTy = llvm::FunctionType::get( 2069 ReservedIDTy, llvm::ArrayRef<llvm::Type *>(ArgTys), false); 2070 // We know the second argument is an integer type, but we may need to cast 2071 // it to i32. 2072 if (Arg1->getType() != Int32Ty) 2073 Arg1 = Builder.CreateZExtOrTrunc(Arg1, Int32Ty); 2074 return RValue::get( 2075 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), {Arg0, Arg1})); 2076 } 2077 // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe commit read and write 2078 // functions 2079 case Builtin::BIcommit_read_pipe: 2080 case Builtin::BIcommit_write_pipe: 2081 case Builtin::BIwork_group_commit_read_pipe: 2082 case Builtin::BIwork_group_commit_write_pipe: 2083 case Builtin::BIsub_group_commit_read_pipe: 2084 case Builtin::BIsub_group_commit_write_pipe: { 2085 const char *Name; 2086 if (BuiltinID == Builtin::BIcommit_read_pipe) 2087 Name = "__commit_read_pipe"; 2088 else if (BuiltinID == Builtin::BIcommit_write_pipe) 2089 Name = "__commit_write_pipe"; 2090 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe) 2091 Name = "__work_group_commit_read_pipe"; 2092 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe) 2093 Name = "__work_group_commit_write_pipe"; 2094 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe) 2095 Name = "__sub_group_commit_read_pipe"; 2096 else 2097 Name = "__sub_group_commit_write_pipe"; 2098 2099 Value *Arg0 = EmitScalarExpr(E->getArg(0)), 2100 *Arg1 = EmitScalarExpr(E->getArg(1)); 2101 2102 // Building the generic function prototype. 2103 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType()}; 2104 llvm::FunctionType *FTy = 2105 llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()), 2106 llvm::ArrayRef<llvm::Type *>(ArgTys), false); 2107 2108 return RValue::get( 2109 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), {Arg0, Arg1})); 2110 } 2111 // OpenCL v2.0 s6.13.16.4 Built-in pipe query functions 2112 case Builtin::BIget_pipe_num_packets: 2113 case Builtin::BIget_pipe_max_packets: { 2114 const char *Name; 2115 if (BuiltinID == Builtin::BIget_pipe_num_packets) 2116 Name = "__get_pipe_num_packets"; 2117 else 2118 Name = "__get_pipe_max_packets"; 2119 2120 // Building the generic function prototype. 2121 Value *Arg0 = EmitScalarExpr(E->getArg(0)); 2122 llvm::Type *ArgTys[] = {Arg0->getType()}; 2123 llvm::FunctionType *FTy = llvm::FunctionType::get( 2124 Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false); 2125 2126 return RValue::get( 2127 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), {Arg0})); 2128 } 2129 2130 // OpenCL v2.0 s6.13.9 - Address space qualifier functions. 2131 case Builtin::BIto_global: 2132 case Builtin::BIto_local: 2133 case Builtin::BIto_private: { 2134 auto Arg0 = EmitScalarExpr(E->getArg(0)); 2135 auto NewArgT = llvm::PointerType::get(Int8Ty, 2136 CGM.getContext().getTargetAddressSpace(LangAS::opencl_generic)); 2137 auto NewRetT = llvm::PointerType::get(Int8Ty, 2138 CGM.getContext().getTargetAddressSpace( 2139 E->getType()->getPointeeType().getAddressSpace())); 2140 auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT}, false); 2141 llvm::Value *NewArg; 2142 if (Arg0->getType()->getPointerAddressSpace() != 2143 NewArgT->getPointerAddressSpace()) 2144 NewArg = Builder.CreateAddrSpaceCast(Arg0, NewArgT); 2145 else 2146 NewArg = Builder.CreateBitOrPointerCast(Arg0, NewArgT); 2147 auto NewCall = Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, 2148 E->getDirectCallee()->getName()), {NewArg}); 2149 return RValue::get(Builder.CreateBitOrPointerCast(NewCall, 2150 ConvertType(E->getType()))); 2151 } 2152 2153 case Builtin::BIprintf: 2154 if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice) 2155 return EmitCUDADevicePrintfCallExpr(E, ReturnValue); 2156 break; 2157 case Builtin::BI__builtin_canonicalize: 2158 case Builtin::BI__builtin_canonicalizef: 2159 case Builtin::BI__builtin_canonicalizel: 2160 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::canonicalize)); 2161 } 2162 2163 // If this is an alias for a lib function (e.g. __builtin_sin), emit 2164 // the call using the normal call path, but using the unmangled 2165 // version of the function name. 2166 if (getContext().BuiltinInfo.isLibFunction(BuiltinID)) 2167 return emitLibraryCall(*this, FD, E, 2168 CGM.getBuiltinLibFunction(FD, BuiltinID)); 2169 2170 // If this is a predefined lib function (e.g. malloc), emit the call 2171 // using exactly the normal call path. 2172 if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID)) 2173 return emitLibraryCall(*this, FD, E, EmitScalarExpr(E->getCallee())); 2174 2175 // Check that a call to a target specific builtin has the correct target 2176 // features. 2177 // This is down here to avoid non-target specific builtins, however, if 2178 // generic builtins start to require generic target features then we 2179 // can move this up to the beginning of the function. 2180 checkTargetFeatures(E, FD); 2181 2182 // See if we have a target specific intrinsic. 2183 const char *Name = getContext().BuiltinInfo.getName(BuiltinID); 2184 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic; 2185 if (const char *Prefix = 2186 llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch())) { 2187 IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix, Name); 2188 // NOTE we dont need to perform a compatibility flag check here since the 2189 // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the 2190 // MS builtins via ALL_MS_LANGUAGES and are filtered earlier. 2191 if (IntrinsicID == Intrinsic::not_intrinsic) 2192 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix, Name); 2193 } 2194 2195 if (IntrinsicID != Intrinsic::not_intrinsic) { 2196 SmallVector<Value*, 16> Args; 2197 2198 // Find out if any arguments are required to be integer constant 2199 // expressions. 2200 unsigned ICEArguments = 0; 2201 ASTContext::GetBuiltinTypeError Error; 2202 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 2203 assert(Error == ASTContext::GE_None && "Should not codegen an error"); 2204 2205 Function *F = CGM.getIntrinsic(IntrinsicID); 2206 llvm::FunctionType *FTy = F->getFunctionType(); 2207 2208 for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) { 2209 Value *ArgValue; 2210 // If this is a normal argument, just emit it as a scalar. 2211 if ((ICEArguments & (1 << i)) == 0) { 2212 ArgValue = EmitScalarExpr(E->getArg(i)); 2213 } else { 2214 // If this is required to be a constant, constant fold it so that we 2215 // know that the generated intrinsic gets a ConstantInt. 2216 llvm::APSInt Result; 2217 bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result,getContext()); 2218 assert(IsConst && "Constant arg isn't actually constant?"); 2219 (void)IsConst; 2220 ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result); 2221 } 2222 2223 // If the intrinsic arg type is different from the builtin arg type 2224 // we need to do a bit cast. 2225 llvm::Type *PTy = FTy->getParamType(i); 2226 if (PTy != ArgValue->getType()) { 2227 assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) && 2228 "Must be able to losslessly bit cast to param"); 2229 ArgValue = Builder.CreateBitCast(ArgValue, PTy); 2230 } 2231 2232 Args.push_back(ArgValue); 2233 } 2234 2235 Value *V = Builder.CreateCall(F, Args); 2236 QualType BuiltinRetType = E->getType(); 2237 2238 llvm::Type *RetTy = VoidTy; 2239 if (!BuiltinRetType->isVoidType()) 2240 RetTy = ConvertType(BuiltinRetType); 2241 2242 if (RetTy != V->getType()) { 2243 assert(V->getType()->canLosslesslyBitCastTo(RetTy) && 2244 "Must be able to losslessly bit cast result type"); 2245 V = Builder.CreateBitCast(V, RetTy); 2246 } 2247 2248 return RValue::get(V); 2249 } 2250 2251 // See if we have a target specific builtin that needs to be lowered. 2252 if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E)) 2253 return RValue::get(V); 2254 2255 ErrorUnsupported(E, "builtin function"); 2256 2257 // Unknown builtin, for now just dump it out and return undef. 2258 return GetUndefRValue(E->getType()); 2259 } 2260 2261 static Value *EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, 2262 unsigned BuiltinID, const CallExpr *E, 2263 llvm::Triple::ArchType Arch) { 2264 switch (Arch) { 2265 case llvm::Triple::arm: 2266 case llvm::Triple::armeb: 2267 case llvm::Triple::thumb: 2268 case llvm::Triple::thumbeb: 2269 return CGF->EmitARMBuiltinExpr(BuiltinID, E); 2270 case llvm::Triple::aarch64: 2271 case llvm::Triple::aarch64_be: 2272 return CGF->EmitAArch64BuiltinExpr(BuiltinID, E); 2273 case llvm::Triple::x86: 2274 case llvm::Triple::x86_64: 2275 return CGF->EmitX86BuiltinExpr(BuiltinID, E); 2276 case llvm::Triple::ppc: 2277 case llvm::Triple::ppc64: 2278 case llvm::Triple::ppc64le: 2279 return CGF->EmitPPCBuiltinExpr(BuiltinID, E); 2280 case llvm::Triple::r600: 2281 case llvm::Triple::amdgcn: 2282 return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E); 2283 case llvm::Triple::systemz: 2284 return CGF->EmitSystemZBuiltinExpr(BuiltinID, E); 2285 case llvm::Triple::nvptx: 2286 case llvm::Triple::nvptx64: 2287 return CGF->EmitNVPTXBuiltinExpr(BuiltinID, E); 2288 case llvm::Triple::wasm32: 2289 case llvm::Triple::wasm64: 2290 return CGF->EmitWebAssemblyBuiltinExpr(BuiltinID, E); 2291 default: 2292 return nullptr; 2293 } 2294 } 2295 2296 Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID, 2297 const CallExpr *E) { 2298 if (getContext().BuiltinInfo.isAuxBuiltinID(BuiltinID)) { 2299 assert(getContext().getAuxTargetInfo() && "Missing aux target info"); 2300 return EmitTargetArchBuiltinExpr( 2301 this, getContext().BuiltinInfo.getAuxBuiltinID(BuiltinID), E, 2302 getContext().getAuxTargetInfo()->getTriple().getArch()); 2303 } 2304 2305 return EmitTargetArchBuiltinExpr(this, BuiltinID, E, 2306 getTarget().getTriple().getArch()); 2307 } 2308 2309 static llvm::VectorType *GetNeonType(CodeGenFunction *CGF, 2310 NeonTypeFlags TypeFlags, 2311 bool V1Ty=false) { 2312 int IsQuad = TypeFlags.isQuad(); 2313 switch (TypeFlags.getEltType()) { 2314 case NeonTypeFlags::Int8: 2315 case NeonTypeFlags::Poly8: 2316 return llvm::VectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad)); 2317 case NeonTypeFlags::Int16: 2318 case NeonTypeFlags::Poly16: 2319 case NeonTypeFlags::Float16: 2320 return llvm::VectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad)); 2321 case NeonTypeFlags::Int32: 2322 return llvm::VectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad)); 2323 case NeonTypeFlags::Int64: 2324 case NeonTypeFlags::Poly64: 2325 return llvm::VectorType::get(CGF->Int64Ty, V1Ty ? 1 : (1 << IsQuad)); 2326 case NeonTypeFlags::Poly128: 2327 // FIXME: i128 and f128 doesn't get fully support in Clang and llvm. 2328 // There is a lot of i128 and f128 API missing. 2329 // so we use v16i8 to represent poly128 and get pattern matched. 2330 return llvm::VectorType::get(CGF->Int8Ty, 16); 2331 case NeonTypeFlags::Float32: 2332 return llvm::VectorType::get(CGF->FloatTy, V1Ty ? 1 : (2 << IsQuad)); 2333 case NeonTypeFlags::Float64: 2334 return llvm::VectorType::get(CGF->DoubleTy, V1Ty ? 1 : (1 << IsQuad)); 2335 } 2336 llvm_unreachable("Unknown vector element type!"); 2337 } 2338 2339 static llvm::VectorType *GetFloatNeonType(CodeGenFunction *CGF, 2340 NeonTypeFlags IntTypeFlags) { 2341 int IsQuad = IntTypeFlags.isQuad(); 2342 switch (IntTypeFlags.getEltType()) { 2343 case NeonTypeFlags::Int32: 2344 return llvm::VectorType::get(CGF->FloatTy, (2 << IsQuad)); 2345 case NeonTypeFlags::Int64: 2346 return llvm::VectorType::get(CGF->DoubleTy, (1 << IsQuad)); 2347 default: 2348 llvm_unreachable("Type can't be converted to floating-point!"); 2349 } 2350 } 2351 2352 Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) { 2353 unsigned nElts = cast<llvm::VectorType>(V->getType())->getNumElements(); 2354 Value* SV = llvm::ConstantVector::getSplat(nElts, C); 2355 return Builder.CreateShuffleVector(V, V, SV, "lane"); 2356 } 2357 2358 Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops, 2359 const char *name, 2360 unsigned shift, bool rightshift) { 2361 unsigned j = 0; 2362 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end(); 2363 ai != ae; ++ai, ++j) 2364 if (shift > 0 && shift == j) 2365 Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift); 2366 else 2367 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name); 2368 2369 return Builder.CreateCall(F, Ops, name); 2370 } 2371 2372 Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty, 2373 bool neg) { 2374 int SV = cast<ConstantInt>(V)->getSExtValue(); 2375 return ConstantInt::get(Ty, neg ? -SV : SV); 2376 } 2377 2378 // \brief Right-shift a vector by a constant. 2379 Value *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift, 2380 llvm::Type *Ty, bool usgn, 2381 const char *name) { 2382 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); 2383 2384 int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue(); 2385 int EltSize = VTy->getScalarSizeInBits(); 2386 2387 Vec = Builder.CreateBitCast(Vec, Ty); 2388 2389 // lshr/ashr are undefined when the shift amount is equal to the vector 2390 // element size. 2391 if (ShiftAmt == EltSize) { 2392 if (usgn) { 2393 // Right-shifting an unsigned value by its size yields 0. 2394 return llvm::ConstantAggregateZero::get(VTy); 2395 } else { 2396 // Right-shifting a signed value by its size is equivalent 2397 // to a shift of size-1. 2398 --ShiftAmt; 2399 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt); 2400 } 2401 } 2402 2403 Shift = EmitNeonShiftVector(Shift, Ty, false); 2404 if (usgn) 2405 return Builder.CreateLShr(Vec, Shift, name); 2406 else 2407 return Builder.CreateAShr(Vec, Shift, name); 2408 } 2409 2410 enum { 2411 AddRetType = (1 << 0), 2412 Add1ArgType = (1 << 1), 2413 Add2ArgTypes = (1 << 2), 2414 2415 VectorizeRetType = (1 << 3), 2416 VectorizeArgTypes = (1 << 4), 2417 2418 InventFloatType = (1 << 5), 2419 UnsignedAlts = (1 << 6), 2420 2421 Use64BitVectors = (1 << 7), 2422 Use128BitVectors = (1 << 8), 2423 2424 Vectorize1ArgType = Add1ArgType | VectorizeArgTypes, 2425 VectorRet = AddRetType | VectorizeRetType, 2426 VectorRetGetArgs01 = 2427 AddRetType | Add2ArgTypes | VectorizeRetType | VectorizeArgTypes, 2428 FpCmpzModifiers = 2429 AddRetType | VectorizeRetType | Add1ArgType | InventFloatType 2430 }; 2431 2432 namespace { 2433 struct NeonIntrinsicInfo { 2434 const char *NameHint; 2435 unsigned BuiltinID; 2436 unsigned LLVMIntrinsic; 2437 unsigned AltLLVMIntrinsic; 2438 unsigned TypeModifier; 2439 2440 bool operator<(unsigned RHSBuiltinID) const { 2441 return BuiltinID < RHSBuiltinID; 2442 } 2443 bool operator<(const NeonIntrinsicInfo &TE) const { 2444 return BuiltinID < TE.BuiltinID; 2445 } 2446 }; 2447 } // end anonymous namespace 2448 2449 #define NEONMAP0(NameBase) \ 2450 { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 } 2451 2452 #define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \ 2453 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \ 2454 Intrinsic::LLVMIntrinsic, 0, TypeModifier } 2455 2456 #define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \ 2457 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \ 2458 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \ 2459 TypeModifier } 2460 2461 static const NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = { 2462 NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts), 2463 NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts), 2464 NEONMAP1(vabs_v, arm_neon_vabs, 0), 2465 NEONMAP1(vabsq_v, arm_neon_vabs, 0), 2466 NEONMAP0(vaddhn_v), 2467 NEONMAP1(vaesdq_v, arm_neon_aesd, 0), 2468 NEONMAP1(vaeseq_v, arm_neon_aese, 0), 2469 NEONMAP1(vaesimcq_v, arm_neon_aesimc, 0), 2470 NEONMAP1(vaesmcq_v, arm_neon_aesmc, 0), 2471 NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType), 2472 NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType), 2473 NEONMAP1(vcage_v, arm_neon_vacge, 0), 2474 NEONMAP1(vcageq_v, arm_neon_vacge, 0), 2475 NEONMAP1(vcagt_v, arm_neon_vacgt, 0), 2476 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0), 2477 NEONMAP1(vcale_v, arm_neon_vacge, 0), 2478 NEONMAP1(vcaleq_v, arm_neon_vacge, 0), 2479 NEONMAP1(vcalt_v, arm_neon_vacgt, 0), 2480 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0), 2481 NEONMAP1(vcls_v, arm_neon_vcls, Add1ArgType), 2482 NEONMAP1(vclsq_v, arm_neon_vcls, Add1ArgType), 2483 NEONMAP1(vclz_v, ctlz, Add1ArgType), 2484 NEONMAP1(vclzq_v, ctlz, Add1ArgType), 2485 NEONMAP1(vcnt_v, ctpop, Add1ArgType), 2486 NEONMAP1(vcntq_v, ctpop, Add1ArgType), 2487 NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0), 2488 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0), 2489 NEONMAP0(vcvt_f32_v), 2490 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), 2491 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0), 2492 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0), 2493 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0), 2494 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0), 2495 NEONMAP0(vcvt_s32_v), 2496 NEONMAP0(vcvt_s64_v), 2497 NEONMAP0(vcvt_u32_v), 2498 NEONMAP0(vcvt_u64_v), 2499 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0), 2500 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0), 2501 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0), 2502 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0), 2503 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0), 2504 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0), 2505 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0), 2506 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0), 2507 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0), 2508 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0), 2509 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0), 2510 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0), 2511 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0), 2512 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0), 2513 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0), 2514 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0), 2515 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0), 2516 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0), 2517 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0), 2518 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0), 2519 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0), 2520 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0), 2521 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0), 2522 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0), 2523 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0), 2524 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0), 2525 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0), 2526 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0), 2527 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0), 2528 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0), 2529 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0), 2530 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0), 2531 NEONMAP0(vcvtq_f32_v), 2532 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), 2533 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0), 2534 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0), 2535 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0), 2536 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0), 2537 NEONMAP0(vcvtq_s32_v), 2538 NEONMAP0(vcvtq_s64_v), 2539 NEONMAP0(vcvtq_u32_v), 2540 NEONMAP0(vcvtq_u64_v), 2541 NEONMAP0(vext_v), 2542 NEONMAP0(vextq_v), 2543 NEONMAP0(vfma_v), 2544 NEONMAP0(vfmaq_v), 2545 NEONMAP2(vhadd_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts), 2546 NEONMAP2(vhaddq_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts), 2547 NEONMAP2(vhsub_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts), 2548 NEONMAP2(vhsubq_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts), 2549 NEONMAP0(vld1_dup_v), 2550 NEONMAP1(vld1_v, arm_neon_vld1, 0), 2551 NEONMAP0(vld1q_dup_v), 2552 NEONMAP1(vld1q_v, arm_neon_vld1, 0), 2553 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0), 2554 NEONMAP1(vld2_v, arm_neon_vld2, 0), 2555 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0), 2556 NEONMAP1(vld2q_v, arm_neon_vld2, 0), 2557 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0), 2558 NEONMAP1(vld3_v, arm_neon_vld3, 0), 2559 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0), 2560 NEONMAP1(vld3q_v, arm_neon_vld3, 0), 2561 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0), 2562 NEONMAP1(vld4_v, arm_neon_vld4, 0), 2563 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0), 2564 NEONMAP1(vld4q_v, arm_neon_vld4, 0), 2565 NEONMAP2(vmax_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts), 2566 NEONMAP1(vmaxnm_v, arm_neon_vmaxnm, Add1ArgType), 2567 NEONMAP1(vmaxnmq_v, arm_neon_vmaxnm, Add1ArgType), 2568 NEONMAP2(vmaxq_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts), 2569 NEONMAP2(vmin_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts), 2570 NEONMAP1(vminnm_v, arm_neon_vminnm, Add1ArgType), 2571 NEONMAP1(vminnmq_v, arm_neon_vminnm, Add1ArgType), 2572 NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts), 2573 NEONMAP0(vmovl_v), 2574 NEONMAP0(vmovn_v), 2575 NEONMAP1(vmul_v, arm_neon_vmulp, Add1ArgType), 2576 NEONMAP0(vmull_v), 2577 NEONMAP1(vmulq_v, arm_neon_vmulp, Add1ArgType), 2578 NEONMAP2(vpadal_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts), 2579 NEONMAP2(vpadalq_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts), 2580 NEONMAP1(vpadd_v, arm_neon_vpadd, Add1ArgType), 2581 NEONMAP2(vpaddl_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts), 2582 NEONMAP2(vpaddlq_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts), 2583 NEONMAP1(vpaddq_v, arm_neon_vpadd, Add1ArgType), 2584 NEONMAP2(vpmax_v, arm_neon_vpmaxu, arm_neon_vpmaxs, Add1ArgType | UnsignedAlts), 2585 NEONMAP2(vpmin_v, arm_neon_vpminu, arm_neon_vpmins, Add1ArgType | UnsignedAlts), 2586 NEONMAP1(vqabs_v, arm_neon_vqabs, Add1ArgType), 2587 NEONMAP1(vqabsq_v, arm_neon_vqabs, Add1ArgType), 2588 NEONMAP2(vqadd_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts), 2589 NEONMAP2(vqaddq_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts), 2590 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, arm_neon_vqadds, 0), 2591 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, arm_neon_vqsubs, 0), 2592 NEONMAP1(vqdmulh_v, arm_neon_vqdmulh, Add1ArgType), 2593 NEONMAP1(vqdmulhq_v, arm_neon_vqdmulh, Add1ArgType), 2594 NEONMAP1(vqdmull_v, arm_neon_vqdmull, Add1ArgType), 2595 NEONMAP2(vqmovn_v, arm_neon_vqmovnu, arm_neon_vqmovns, Add1ArgType | UnsignedAlts), 2596 NEONMAP1(vqmovun_v, arm_neon_vqmovnsu, Add1ArgType), 2597 NEONMAP1(vqneg_v, arm_neon_vqneg, Add1ArgType), 2598 NEONMAP1(vqnegq_v, arm_neon_vqneg, Add1ArgType), 2599 NEONMAP1(vqrdmulh_v, arm_neon_vqrdmulh, Add1ArgType), 2600 NEONMAP1(vqrdmulhq_v, arm_neon_vqrdmulh, Add1ArgType), 2601 NEONMAP2(vqrshl_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts), 2602 NEONMAP2(vqrshlq_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts), 2603 NEONMAP2(vqshl_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts), 2604 NEONMAP2(vqshl_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts), 2605 NEONMAP2(vqshlq_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts), 2606 NEONMAP2(vqshlq_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts), 2607 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0), 2608 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0), 2609 NEONMAP2(vqsub_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts), 2610 NEONMAP2(vqsubq_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts), 2611 NEONMAP1(vraddhn_v, arm_neon_vraddhn, Add1ArgType), 2612 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0), 2613 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0), 2614 NEONMAP1(vrecps_v, arm_neon_vrecps, Add1ArgType), 2615 NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType), 2616 NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts), 2617 NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts), 2618 NEONMAP1(vrnd_v, arm_neon_vrintz, Add1ArgType), 2619 NEONMAP1(vrnda_v, arm_neon_vrinta, Add1ArgType), 2620 NEONMAP1(vrndaq_v, arm_neon_vrinta, Add1ArgType), 2621 NEONMAP1(vrndm_v, arm_neon_vrintm, Add1ArgType), 2622 NEONMAP1(vrndmq_v, arm_neon_vrintm, Add1ArgType), 2623 NEONMAP1(vrndn_v, arm_neon_vrintn, Add1ArgType), 2624 NEONMAP1(vrndnq_v, arm_neon_vrintn, Add1ArgType), 2625 NEONMAP1(vrndp_v, arm_neon_vrintp, Add1ArgType), 2626 NEONMAP1(vrndpq_v, arm_neon_vrintp, Add1ArgType), 2627 NEONMAP1(vrndq_v, arm_neon_vrintz, Add1ArgType), 2628 NEONMAP1(vrndx_v, arm_neon_vrintx, Add1ArgType), 2629 NEONMAP1(vrndxq_v, arm_neon_vrintx, Add1ArgType), 2630 NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts), 2631 NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts), 2632 NEONMAP2(vrshr_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts), 2633 NEONMAP2(vrshrq_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts), 2634 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0), 2635 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0), 2636 NEONMAP1(vrsqrts_v, arm_neon_vrsqrts, Add1ArgType), 2637 NEONMAP1(vrsqrtsq_v, arm_neon_vrsqrts, Add1ArgType), 2638 NEONMAP1(vrsubhn_v, arm_neon_vrsubhn, Add1ArgType), 2639 NEONMAP1(vsha1su0q_v, arm_neon_sha1su0, 0), 2640 NEONMAP1(vsha1su1q_v, arm_neon_sha1su1, 0), 2641 NEONMAP1(vsha256h2q_v, arm_neon_sha256h2, 0), 2642 NEONMAP1(vsha256hq_v, arm_neon_sha256h, 0), 2643 NEONMAP1(vsha256su0q_v, arm_neon_sha256su0, 0), 2644 NEONMAP1(vsha256su1q_v, arm_neon_sha256su1, 0), 2645 NEONMAP0(vshl_n_v), 2646 NEONMAP2(vshl_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts), 2647 NEONMAP0(vshll_n_v), 2648 NEONMAP0(vshlq_n_v), 2649 NEONMAP2(vshlq_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts), 2650 NEONMAP0(vshr_n_v), 2651 NEONMAP0(vshrn_n_v), 2652 NEONMAP0(vshrq_n_v), 2653 NEONMAP1(vst1_v, arm_neon_vst1, 0), 2654 NEONMAP1(vst1q_v, arm_neon_vst1, 0), 2655 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0), 2656 NEONMAP1(vst2_v, arm_neon_vst2, 0), 2657 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0), 2658 NEONMAP1(vst2q_v, arm_neon_vst2, 0), 2659 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0), 2660 NEONMAP1(vst3_v, arm_neon_vst3, 0), 2661 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0), 2662 NEONMAP1(vst3q_v, arm_neon_vst3, 0), 2663 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0), 2664 NEONMAP1(vst4_v, arm_neon_vst4, 0), 2665 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0), 2666 NEONMAP1(vst4q_v, arm_neon_vst4, 0), 2667 NEONMAP0(vsubhn_v), 2668 NEONMAP0(vtrn_v), 2669 NEONMAP0(vtrnq_v), 2670 NEONMAP0(vtst_v), 2671 NEONMAP0(vtstq_v), 2672 NEONMAP0(vuzp_v), 2673 NEONMAP0(vuzpq_v), 2674 NEONMAP0(vzip_v), 2675 NEONMAP0(vzipq_v) 2676 }; 2677 2678 static const NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = { 2679 NEONMAP1(vabs_v, aarch64_neon_abs, 0), 2680 NEONMAP1(vabsq_v, aarch64_neon_abs, 0), 2681 NEONMAP0(vaddhn_v), 2682 NEONMAP1(vaesdq_v, aarch64_crypto_aesd, 0), 2683 NEONMAP1(vaeseq_v, aarch64_crypto_aese, 0), 2684 NEONMAP1(vaesimcq_v, aarch64_crypto_aesimc, 0), 2685 NEONMAP1(vaesmcq_v, aarch64_crypto_aesmc, 0), 2686 NEONMAP1(vcage_v, aarch64_neon_facge, 0), 2687 NEONMAP1(vcageq_v, aarch64_neon_facge, 0), 2688 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0), 2689 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0), 2690 NEONMAP1(vcale_v, aarch64_neon_facge, 0), 2691 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0), 2692 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0), 2693 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0), 2694 NEONMAP1(vcls_v, aarch64_neon_cls, Add1ArgType), 2695 NEONMAP1(vclsq_v, aarch64_neon_cls, Add1ArgType), 2696 NEONMAP1(vclz_v, ctlz, Add1ArgType), 2697 NEONMAP1(vclzq_v, ctlz, Add1ArgType), 2698 NEONMAP1(vcnt_v, ctpop, Add1ArgType), 2699 NEONMAP1(vcntq_v, ctpop, Add1ArgType), 2700 NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0), 2701 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0), 2702 NEONMAP0(vcvt_f32_v), 2703 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 2704 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 2705 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0), 2706 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0), 2707 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0), 2708 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0), 2709 NEONMAP0(vcvtq_f32_v), 2710 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 2711 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 2712 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0), 2713 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0), 2714 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0), 2715 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0), 2716 NEONMAP1(vcvtx_f32_v, aarch64_neon_fcvtxn, AddRetType | Add1ArgType), 2717 NEONMAP0(vext_v), 2718 NEONMAP0(vextq_v), 2719 NEONMAP0(vfma_v), 2720 NEONMAP0(vfmaq_v), 2721 NEONMAP2(vhadd_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts), 2722 NEONMAP2(vhaddq_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts), 2723 NEONMAP2(vhsub_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts), 2724 NEONMAP2(vhsubq_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts), 2725 NEONMAP0(vmovl_v), 2726 NEONMAP0(vmovn_v), 2727 NEONMAP1(vmul_v, aarch64_neon_pmul, Add1ArgType), 2728 NEONMAP1(vmulq_v, aarch64_neon_pmul, Add1ArgType), 2729 NEONMAP1(vpadd_v, aarch64_neon_addp, Add1ArgType), 2730 NEONMAP2(vpaddl_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts), 2731 NEONMAP2(vpaddlq_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts), 2732 NEONMAP1(vpaddq_v, aarch64_neon_addp, Add1ArgType), 2733 NEONMAP1(vqabs_v, aarch64_neon_sqabs, Add1ArgType), 2734 NEONMAP1(vqabsq_v, aarch64_neon_sqabs, Add1ArgType), 2735 NEONMAP2(vqadd_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts), 2736 NEONMAP2(vqaddq_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts), 2737 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0), 2738 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0), 2739 NEONMAP1(vqdmulh_v, aarch64_neon_sqdmulh, Add1ArgType), 2740 NEONMAP1(vqdmulhq_v, aarch64_neon_sqdmulh, Add1ArgType), 2741 NEONMAP1(vqdmull_v, aarch64_neon_sqdmull, Add1ArgType), 2742 NEONMAP2(vqmovn_v, aarch64_neon_uqxtn, aarch64_neon_sqxtn, Add1ArgType | UnsignedAlts), 2743 NEONMAP1(vqmovun_v, aarch64_neon_sqxtun, Add1ArgType), 2744 NEONMAP1(vqneg_v, aarch64_neon_sqneg, Add1ArgType), 2745 NEONMAP1(vqnegq_v, aarch64_neon_sqneg, Add1ArgType), 2746 NEONMAP1(vqrdmulh_v, aarch64_neon_sqrdmulh, Add1ArgType), 2747 NEONMAP1(vqrdmulhq_v, aarch64_neon_sqrdmulh, Add1ArgType), 2748 NEONMAP2(vqrshl_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts), 2749 NEONMAP2(vqrshlq_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts), 2750 NEONMAP2(vqshl_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl, UnsignedAlts), 2751 NEONMAP2(vqshl_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts), 2752 NEONMAP2(vqshlq_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl,UnsignedAlts), 2753 NEONMAP2(vqshlq_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts), 2754 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0), 2755 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0), 2756 NEONMAP2(vqsub_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts), 2757 NEONMAP2(vqsubq_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts), 2758 NEONMAP1(vraddhn_v, aarch64_neon_raddhn, Add1ArgType), 2759 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0), 2760 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0), 2761 NEONMAP1(vrecps_v, aarch64_neon_frecps, Add1ArgType), 2762 NEONMAP1(vrecpsq_v, aarch64_neon_frecps, Add1ArgType), 2763 NEONMAP2(vrhadd_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts), 2764 NEONMAP2(vrhaddq_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts), 2765 NEONMAP2(vrshl_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts), 2766 NEONMAP2(vrshlq_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts), 2767 NEONMAP2(vrshr_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts), 2768 NEONMAP2(vrshrq_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts), 2769 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0), 2770 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0), 2771 NEONMAP1(vrsqrts_v, aarch64_neon_frsqrts, Add1ArgType), 2772 NEONMAP1(vrsqrtsq_v, aarch64_neon_frsqrts, Add1ArgType), 2773 NEONMAP1(vrsubhn_v, aarch64_neon_rsubhn, Add1ArgType), 2774 NEONMAP1(vsha1su0q_v, aarch64_crypto_sha1su0, 0), 2775 NEONMAP1(vsha1su1q_v, aarch64_crypto_sha1su1, 0), 2776 NEONMAP1(vsha256h2q_v, aarch64_crypto_sha256h2, 0), 2777 NEONMAP1(vsha256hq_v, aarch64_crypto_sha256h, 0), 2778 NEONMAP1(vsha256su0q_v, aarch64_crypto_sha256su0, 0), 2779 NEONMAP1(vsha256su1q_v, aarch64_crypto_sha256su1, 0), 2780 NEONMAP0(vshl_n_v), 2781 NEONMAP2(vshl_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts), 2782 NEONMAP0(vshll_n_v), 2783 NEONMAP0(vshlq_n_v), 2784 NEONMAP2(vshlq_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts), 2785 NEONMAP0(vshr_n_v), 2786 NEONMAP0(vshrn_n_v), 2787 NEONMAP0(vshrq_n_v), 2788 NEONMAP0(vsubhn_v), 2789 NEONMAP0(vtst_v), 2790 NEONMAP0(vtstq_v), 2791 }; 2792 2793 static const NeonIntrinsicInfo AArch64SISDIntrinsicMap[] = { 2794 NEONMAP1(vabdd_f64, aarch64_sisd_fabd, Add1ArgType), 2795 NEONMAP1(vabds_f32, aarch64_sisd_fabd, Add1ArgType), 2796 NEONMAP1(vabsd_s64, aarch64_neon_abs, Add1ArgType), 2797 NEONMAP1(vaddlv_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType), 2798 NEONMAP1(vaddlv_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType), 2799 NEONMAP1(vaddlvq_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType), 2800 NEONMAP1(vaddlvq_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType), 2801 NEONMAP1(vaddv_f32, aarch64_neon_faddv, AddRetType | Add1ArgType), 2802 NEONMAP1(vaddv_s32, aarch64_neon_saddv, AddRetType | Add1ArgType), 2803 NEONMAP1(vaddv_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType), 2804 NEONMAP1(vaddvq_f32, aarch64_neon_faddv, AddRetType | Add1ArgType), 2805 NEONMAP1(vaddvq_f64, aarch64_neon_faddv, AddRetType | Add1ArgType), 2806 NEONMAP1(vaddvq_s32, aarch64_neon_saddv, AddRetType | Add1ArgType), 2807 NEONMAP1(vaddvq_s64, aarch64_neon_saddv, AddRetType | Add1ArgType), 2808 NEONMAP1(vaddvq_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType), 2809 NEONMAP1(vaddvq_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 2810 NEONMAP1(vcaged_f64, aarch64_neon_facge, AddRetType | Add1ArgType), 2811 NEONMAP1(vcages_f32, aarch64_neon_facge, AddRetType | Add1ArgType), 2812 NEONMAP1(vcagtd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType), 2813 NEONMAP1(vcagts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType), 2814 NEONMAP1(vcaled_f64, aarch64_neon_facge, AddRetType | Add1ArgType), 2815 NEONMAP1(vcales_f32, aarch64_neon_facge, AddRetType | Add1ArgType), 2816 NEONMAP1(vcaltd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType), 2817 NEONMAP1(vcalts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType), 2818 NEONMAP1(vcvtad_s64_f64, aarch64_neon_fcvtas, AddRetType | Add1ArgType), 2819 NEONMAP1(vcvtad_u64_f64, aarch64_neon_fcvtau, AddRetType | Add1ArgType), 2820 NEONMAP1(vcvtas_s32_f32, aarch64_neon_fcvtas, AddRetType | Add1ArgType), 2821 NEONMAP1(vcvtas_u32_f32, aarch64_neon_fcvtau, AddRetType | Add1ArgType), 2822 NEONMAP1(vcvtd_n_f64_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType), 2823 NEONMAP1(vcvtd_n_f64_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType), 2824 NEONMAP1(vcvtd_n_s64_f64, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType), 2825 NEONMAP1(vcvtd_n_u64_f64, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType), 2826 NEONMAP1(vcvtmd_s64_f64, aarch64_neon_fcvtms, AddRetType | Add1ArgType), 2827 NEONMAP1(vcvtmd_u64_f64, aarch64_neon_fcvtmu, AddRetType | Add1ArgType), 2828 NEONMAP1(vcvtms_s32_f32, aarch64_neon_fcvtms, AddRetType | Add1ArgType), 2829 NEONMAP1(vcvtms_u32_f32, aarch64_neon_fcvtmu, AddRetType | Add1ArgType), 2830 NEONMAP1(vcvtnd_s64_f64, aarch64_neon_fcvtns, AddRetType | Add1ArgType), 2831 NEONMAP1(vcvtnd_u64_f64, aarch64_neon_fcvtnu, AddRetType | Add1ArgType), 2832 NEONMAP1(vcvtns_s32_f32, aarch64_neon_fcvtns, AddRetType | Add1ArgType), 2833 NEONMAP1(vcvtns_u32_f32, aarch64_neon_fcvtnu, AddRetType | Add1ArgType), 2834 NEONMAP1(vcvtpd_s64_f64, aarch64_neon_fcvtps, AddRetType | Add1ArgType), 2835 NEONMAP1(vcvtpd_u64_f64, aarch64_neon_fcvtpu, AddRetType | Add1ArgType), 2836 NEONMAP1(vcvtps_s32_f32, aarch64_neon_fcvtps, AddRetType | Add1ArgType), 2837 NEONMAP1(vcvtps_u32_f32, aarch64_neon_fcvtpu, AddRetType | Add1ArgType), 2838 NEONMAP1(vcvts_n_f32_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType), 2839 NEONMAP1(vcvts_n_f32_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType), 2840 NEONMAP1(vcvts_n_s32_f32, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType), 2841 NEONMAP1(vcvts_n_u32_f32, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType), 2842 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0), 2843 NEONMAP1(vmaxnmv_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 2844 NEONMAP1(vmaxnmvq_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 2845 NEONMAP1(vmaxnmvq_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 2846 NEONMAP1(vmaxv_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 2847 NEONMAP1(vmaxv_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType), 2848 NEONMAP1(vmaxv_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType), 2849 NEONMAP1(vmaxvq_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 2850 NEONMAP1(vmaxvq_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 2851 NEONMAP1(vmaxvq_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType), 2852 NEONMAP1(vmaxvq_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType), 2853 NEONMAP1(vminnmv_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 2854 NEONMAP1(vminnmvq_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 2855 NEONMAP1(vminnmvq_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 2856 NEONMAP1(vminv_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 2857 NEONMAP1(vminv_s32, aarch64_neon_sminv, AddRetType | Add1ArgType), 2858 NEONMAP1(vminv_u32, aarch64_neon_uminv, AddRetType | Add1ArgType), 2859 NEONMAP1(vminvq_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 2860 NEONMAP1(vminvq_f64, aarch64_neon_fminv, AddRetType | Add1ArgType), 2861 NEONMAP1(vminvq_s32, aarch64_neon_sminv, AddRetType | Add1ArgType), 2862 NEONMAP1(vminvq_u32, aarch64_neon_uminv, AddRetType | Add1ArgType), 2863 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0), 2864 NEONMAP1(vmulxd_f64, aarch64_neon_fmulx, Add1ArgType), 2865 NEONMAP1(vmulxs_f32, aarch64_neon_fmulx, Add1ArgType), 2866 NEONMAP1(vpaddd_s64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 2867 NEONMAP1(vpaddd_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 2868 NEONMAP1(vpmaxnmqd_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 2869 NEONMAP1(vpmaxnms_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 2870 NEONMAP1(vpmaxqd_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 2871 NEONMAP1(vpmaxs_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 2872 NEONMAP1(vpminnmqd_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 2873 NEONMAP1(vpminnms_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 2874 NEONMAP1(vpminqd_f64, aarch64_neon_fminv, AddRetType | Add1ArgType), 2875 NEONMAP1(vpmins_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 2876 NEONMAP1(vqabsb_s8, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors), 2877 NEONMAP1(vqabsd_s64, aarch64_neon_sqabs, Add1ArgType), 2878 NEONMAP1(vqabsh_s16, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors), 2879 NEONMAP1(vqabss_s32, aarch64_neon_sqabs, Add1ArgType), 2880 NEONMAP1(vqaddb_s8, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors), 2881 NEONMAP1(vqaddb_u8, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors), 2882 NEONMAP1(vqaddd_s64, aarch64_neon_sqadd, Add1ArgType), 2883 NEONMAP1(vqaddd_u64, aarch64_neon_uqadd, Add1ArgType), 2884 NEONMAP1(vqaddh_s16, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors), 2885 NEONMAP1(vqaddh_u16, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors), 2886 NEONMAP1(vqadds_s32, aarch64_neon_sqadd, Add1ArgType), 2887 NEONMAP1(vqadds_u32, aarch64_neon_uqadd, Add1ArgType), 2888 NEONMAP1(vqdmulhh_s16, aarch64_neon_sqdmulh, Vectorize1ArgType | Use64BitVectors), 2889 NEONMAP1(vqdmulhs_s32, aarch64_neon_sqdmulh, Add1ArgType), 2890 NEONMAP1(vqdmullh_s16, aarch64_neon_sqdmull, VectorRet | Use128BitVectors), 2891 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0), 2892 NEONMAP1(vqmovnd_s64, aarch64_neon_scalar_sqxtn, AddRetType | Add1ArgType), 2893 NEONMAP1(vqmovnd_u64, aarch64_neon_scalar_uqxtn, AddRetType | Add1ArgType), 2894 NEONMAP1(vqmovnh_s16, aarch64_neon_sqxtn, VectorRet | Use64BitVectors), 2895 NEONMAP1(vqmovnh_u16, aarch64_neon_uqxtn, VectorRet | Use64BitVectors), 2896 NEONMAP1(vqmovns_s32, aarch64_neon_sqxtn, VectorRet | Use64BitVectors), 2897 NEONMAP1(vqmovns_u32, aarch64_neon_uqxtn, VectorRet | Use64BitVectors), 2898 NEONMAP1(vqmovund_s64, aarch64_neon_scalar_sqxtun, AddRetType | Add1ArgType), 2899 NEONMAP1(vqmovunh_s16, aarch64_neon_sqxtun, VectorRet | Use64BitVectors), 2900 NEONMAP1(vqmovuns_s32, aarch64_neon_sqxtun, VectorRet | Use64BitVectors), 2901 NEONMAP1(vqnegb_s8, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors), 2902 NEONMAP1(vqnegd_s64, aarch64_neon_sqneg, Add1ArgType), 2903 NEONMAP1(vqnegh_s16, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors), 2904 NEONMAP1(vqnegs_s32, aarch64_neon_sqneg, Add1ArgType), 2905 NEONMAP1(vqrdmulhh_s16, aarch64_neon_sqrdmulh, Vectorize1ArgType | Use64BitVectors), 2906 NEONMAP1(vqrdmulhs_s32, aarch64_neon_sqrdmulh, Add1ArgType), 2907 NEONMAP1(vqrshlb_s8, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors), 2908 NEONMAP1(vqrshlb_u8, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors), 2909 NEONMAP1(vqrshld_s64, aarch64_neon_sqrshl, Add1ArgType), 2910 NEONMAP1(vqrshld_u64, aarch64_neon_uqrshl, Add1ArgType), 2911 NEONMAP1(vqrshlh_s16, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors), 2912 NEONMAP1(vqrshlh_u16, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors), 2913 NEONMAP1(vqrshls_s32, aarch64_neon_sqrshl, Add1ArgType), 2914 NEONMAP1(vqrshls_u32, aarch64_neon_uqrshl, Add1ArgType), 2915 NEONMAP1(vqrshrnd_n_s64, aarch64_neon_sqrshrn, AddRetType), 2916 NEONMAP1(vqrshrnd_n_u64, aarch64_neon_uqrshrn, AddRetType), 2917 NEONMAP1(vqrshrnh_n_s16, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors), 2918 NEONMAP1(vqrshrnh_n_u16, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors), 2919 NEONMAP1(vqrshrns_n_s32, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors), 2920 NEONMAP1(vqrshrns_n_u32, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors), 2921 NEONMAP1(vqrshrund_n_s64, aarch64_neon_sqrshrun, AddRetType), 2922 NEONMAP1(vqrshrunh_n_s16, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors), 2923 NEONMAP1(vqrshruns_n_s32, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors), 2924 NEONMAP1(vqshlb_n_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 2925 NEONMAP1(vqshlb_n_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 2926 NEONMAP1(vqshlb_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 2927 NEONMAP1(vqshlb_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 2928 NEONMAP1(vqshld_s64, aarch64_neon_sqshl, Add1ArgType), 2929 NEONMAP1(vqshld_u64, aarch64_neon_uqshl, Add1ArgType), 2930 NEONMAP1(vqshlh_n_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 2931 NEONMAP1(vqshlh_n_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 2932 NEONMAP1(vqshlh_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 2933 NEONMAP1(vqshlh_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 2934 NEONMAP1(vqshls_n_s32, aarch64_neon_sqshl, Add1ArgType), 2935 NEONMAP1(vqshls_n_u32, aarch64_neon_uqshl, Add1ArgType), 2936 NEONMAP1(vqshls_s32, aarch64_neon_sqshl, Add1ArgType), 2937 NEONMAP1(vqshls_u32, aarch64_neon_uqshl, Add1ArgType), 2938 NEONMAP1(vqshlub_n_s8, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors), 2939 NEONMAP1(vqshluh_n_s16, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors), 2940 NEONMAP1(vqshlus_n_s32, aarch64_neon_sqshlu, Add1ArgType), 2941 NEONMAP1(vqshrnd_n_s64, aarch64_neon_sqshrn, AddRetType), 2942 NEONMAP1(vqshrnd_n_u64, aarch64_neon_uqshrn, AddRetType), 2943 NEONMAP1(vqshrnh_n_s16, aarch64_neon_sqshrn, VectorRet | Use64BitVectors), 2944 NEONMAP1(vqshrnh_n_u16, aarch64_neon_uqshrn, VectorRet | Use64BitVectors), 2945 NEONMAP1(vqshrns_n_s32, aarch64_neon_sqshrn, VectorRet | Use64BitVectors), 2946 NEONMAP1(vqshrns_n_u32, aarch64_neon_uqshrn, VectorRet | Use64BitVectors), 2947 NEONMAP1(vqshrund_n_s64, aarch64_neon_sqshrun, AddRetType), 2948 NEONMAP1(vqshrunh_n_s16, aarch64_neon_sqshrun, VectorRet | Use64BitVectors), 2949 NEONMAP1(vqshruns_n_s32, aarch64_neon_sqshrun, VectorRet | Use64BitVectors), 2950 NEONMAP1(vqsubb_s8, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors), 2951 NEONMAP1(vqsubb_u8, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors), 2952 NEONMAP1(vqsubd_s64, aarch64_neon_sqsub, Add1ArgType), 2953 NEONMAP1(vqsubd_u64, aarch64_neon_uqsub, Add1ArgType), 2954 NEONMAP1(vqsubh_s16, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors), 2955 NEONMAP1(vqsubh_u16, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors), 2956 NEONMAP1(vqsubs_s32, aarch64_neon_sqsub, Add1ArgType), 2957 NEONMAP1(vqsubs_u32, aarch64_neon_uqsub, Add1ArgType), 2958 NEONMAP1(vrecped_f64, aarch64_neon_frecpe, Add1ArgType), 2959 NEONMAP1(vrecpes_f32, aarch64_neon_frecpe, Add1ArgType), 2960 NEONMAP1(vrecpxd_f64, aarch64_neon_frecpx, Add1ArgType), 2961 NEONMAP1(vrecpxs_f32, aarch64_neon_frecpx, Add1ArgType), 2962 NEONMAP1(vrshld_s64, aarch64_neon_srshl, Add1ArgType), 2963 NEONMAP1(vrshld_u64, aarch64_neon_urshl, Add1ArgType), 2964 NEONMAP1(vrsqrted_f64, aarch64_neon_frsqrte, Add1ArgType), 2965 NEONMAP1(vrsqrtes_f32, aarch64_neon_frsqrte, Add1ArgType), 2966 NEONMAP1(vrsqrtsd_f64, aarch64_neon_frsqrts, Add1ArgType), 2967 NEONMAP1(vrsqrtss_f32, aarch64_neon_frsqrts, Add1ArgType), 2968 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0), 2969 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0), 2970 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0), 2971 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0), 2972 NEONMAP1(vshld_s64, aarch64_neon_sshl, Add1ArgType), 2973 NEONMAP1(vshld_u64, aarch64_neon_ushl, Add1ArgType), 2974 NEONMAP1(vslid_n_s64, aarch64_neon_vsli, Vectorize1ArgType), 2975 NEONMAP1(vslid_n_u64, aarch64_neon_vsli, Vectorize1ArgType), 2976 NEONMAP1(vsqaddb_u8, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors), 2977 NEONMAP1(vsqaddd_u64, aarch64_neon_usqadd, Add1ArgType), 2978 NEONMAP1(vsqaddh_u16, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors), 2979 NEONMAP1(vsqadds_u32, aarch64_neon_usqadd, Add1ArgType), 2980 NEONMAP1(vsrid_n_s64, aarch64_neon_vsri, Vectorize1ArgType), 2981 NEONMAP1(vsrid_n_u64, aarch64_neon_vsri, Vectorize1ArgType), 2982 NEONMAP1(vuqaddb_s8, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors), 2983 NEONMAP1(vuqaddd_s64, aarch64_neon_suqadd, Add1ArgType), 2984 NEONMAP1(vuqaddh_s16, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors), 2985 NEONMAP1(vuqadds_s32, aarch64_neon_suqadd, Add1ArgType), 2986 }; 2987 2988 #undef NEONMAP0 2989 #undef NEONMAP1 2990 #undef NEONMAP2 2991 2992 static bool NEONSIMDIntrinsicsProvenSorted = false; 2993 2994 static bool AArch64SIMDIntrinsicsProvenSorted = false; 2995 static bool AArch64SISDIntrinsicsProvenSorted = false; 2996 2997 2998 static const NeonIntrinsicInfo * 2999 findNeonIntrinsicInMap(ArrayRef<NeonIntrinsicInfo> IntrinsicMap, 3000 unsigned BuiltinID, bool &MapProvenSorted) { 3001 3002 #ifndef NDEBUG 3003 if (!MapProvenSorted) { 3004 assert(std::is_sorted(std::begin(IntrinsicMap), std::end(IntrinsicMap))); 3005 MapProvenSorted = true; 3006 } 3007 #endif 3008 3009 const NeonIntrinsicInfo *Builtin = 3010 std::lower_bound(IntrinsicMap.begin(), IntrinsicMap.end(), BuiltinID); 3011 3012 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID) 3013 return Builtin; 3014 3015 return nullptr; 3016 } 3017 3018 Function *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID, 3019 unsigned Modifier, 3020 llvm::Type *ArgType, 3021 const CallExpr *E) { 3022 int VectorSize = 0; 3023 if (Modifier & Use64BitVectors) 3024 VectorSize = 64; 3025 else if (Modifier & Use128BitVectors) 3026 VectorSize = 128; 3027 3028 // Return type. 3029 SmallVector<llvm::Type *, 3> Tys; 3030 if (Modifier & AddRetType) { 3031 llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext())); 3032 if (Modifier & VectorizeRetType) 3033 Ty = llvm::VectorType::get( 3034 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1); 3035 3036 Tys.push_back(Ty); 3037 } 3038 3039 // Arguments. 3040 if (Modifier & VectorizeArgTypes) { 3041 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1; 3042 ArgType = llvm::VectorType::get(ArgType, Elts); 3043 } 3044 3045 if (Modifier & (Add1ArgType | Add2ArgTypes)) 3046 Tys.push_back(ArgType); 3047 3048 if (Modifier & Add2ArgTypes) 3049 Tys.push_back(ArgType); 3050 3051 if (Modifier & InventFloatType) 3052 Tys.push_back(FloatTy); 3053 3054 return CGM.getIntrinsic(IntrinsicID, Tys); 3055 } 3056 3057 static Value *EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, 3058 const NeonIntrinsicInfo &SISDInfo, 3059 SmallVectorImpl<Value *> &Ops, 3060 const CallExpr *E) { 3061 unsigned BuiltinID = SISDInfo.BuiltinID; 3062 unsigned int Int = SISDInfo.LLVMIntrinsic; 3063 unsigned Modifier = SISDInfo.TypeModifier; 3064 const char *s = SISDInfo.NameHint; 3065 3066 switch (BuiltinID) { 3067 case NEON::BI__builtin_neon_vcled_s64: 3068 case NEON::BI__builtin_neon_vcled_u64: 3069 case NEON::BI__builtin_neon_vcles_f32: 3070 case NEON::BI__builtin_neon_vcled_f64: 3071 case NEON::BI__builtin_neon_vcltd_s64: 3072 case NEON::BI__builtin_neon_vcltd_u64: 3073 case NEON::BI__builtin_neon_vclts_f32: 3074 case NEON::BI__builtin_neon_vcltd_f64: 3075 case NEON::BI__builtin_neon_vcales_f32: 3076 case NEON::BI__builtin_neon_vcaled_f64: 3077 case NEON::BI__builtin_neon_vcalts_f32: 3078 case NEON::BI__builtin_neon_vcaltd_f64: 3079 // Only one direction of comparisons actually exist, cmle is actually a cmge 3080 // with swapped operands. The table gives us the right intrinsic but we 3081 // still need to do the swap. 3082 std::swap(Ops[0], Ops[1]); 3083 break; 3084 } 3085 3086 assert(Int && "Generic code assumes a valid intrinsic"); 3087 3088 // Determine the type(s) of this overloaded AArch64 intrinsic. 3089 const Expr *Arg = E->getArg(0); 3090 llvm::Type *ArgTy = CGF.ConvertType(Arg->getType()); 3091 Function *F = CGF.LookupNeonLLVMIntrinsic(Int, Modifier, ArgTy, E); 3092 3093 int j = 0; 3094 ConstantInt *C0 = ConstantInt::get(CGF.SizeTy, 0); 3095 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end(); 3096 ai != ae; ++ai, ++j) { 3097 llvm::Type *ArgTy = ai->getType(); 3098 if (Ops[j]->getType()->getPrimitiveSizeInBits() == 3099 ArgTy->getPrimitiveSizeInBits()) 3100 continue; 3101 3102 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy()); 3103 // The constant argument to an _n_ intrinsic always has Int32Ty, so truncate 3104 // it before inserting. 3105 Ops[j] = 3106 CGF.Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType()); 3107 Ops[j] = 3108 CGF.Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0); 3109 } 3110 3111 Value *Result = CGF.EmitNeonCall(F, Ops, s); 3112 llvm::Type *ResultType = CGF.ConvertType(E->getType()); 3113 if (ResultType->getPrimitiveSizeInBits() < 3114 Result->getType()->getPrimitiveSizeInBits()) 3115 return CGF.Builder.CreateExtractElement(Result, C0); 3116 3117 return CGF.Builder.CreateBitCast(Result, ResultType, s); 3118 } 3119 3120 Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( 3121 unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, 3122 const char *NameHint, unsigned Modifier, const CallExpr *E, 3123 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1) { 3124 // Get the last argument, which specifies the vector type. 3125 llvm::APSInt NeonTypeConst; 3126 const Expr *Arg = E->getArg(E->getNumArgs() - 1); 3127 if (!Arg->isIntegerConstantExpr(NeonTypeConst, getContext())) 3128 return nullptr; 3129 3130 // Determine the type of this overloaded NEON intrinsic. 3131 NeonTypeFlags Type(NeonTypeConst.getZExtValue()); 3132 bool Usgn = Type.isUnsigned(); 3133 bool Quad = Type.isQuad(); 3134 3135 llvm::VectorType *VTy = GetNeonType(this, Type); 3136 llvm::Type *Ty = VTy; 3137 if (!Ty) 3138 return nullptr; 3139 3140 auto getAlignmentValue32 = [&](Address addr) -> Value* { 3141 return Builder.getInt32(addr.getAlignment().getQuantity()); 3142 }; 3143 3144 unsigned Int = LLVMIntrinsic; 3145 if ((Modifier & UnsignedAlts) && !Usgn) 3146 Int = AltLLVMIntrinsic; 3147 3148 switch (BuiltinID) { 3149 default: break; 3150 case NEON::BI__builtin_neon_vabs_v: 3151 case NEON::BI__builtin_neon_vabsq_v: 3152 if (VTy->getElementType()->isFloatingPointTy()) 3153 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs"); 3154 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs"); 3155 case NEON::BI__builtin_neon_vaddhn_v: { 3156 llvm::VectorType *SrcTy = 3157 llvm::VectorType::getExtendedElementVectorType(VTy); 3158 3159 // %sum = add <4 x i32> %lhs, %rhs 3160 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 3161 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); 3162 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn"); 3163 3164 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16> 3165 Constant *ShiftAmt = 3166 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2); 3167 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn"); 3168 3169 // %res = trunc <4 x i32> %high to <4 x i16> 3170 return Builder.CreateTrunc(Ops[0], VTy, "vaddhn"); 3171 } 3172 case NEON::BI__builtin_neon_vcale_v: 3173 case NEON::BI__builtin_neon_vcaleq_v: 3174 case NEON::BI__builtin_neon_vcalt_v: 3175 case NEON::BI__builtin_neon_vcaltq_v: 3176 std::swap(Ops[0], Ops[1]); 3177 case NEON::BI__builtin_neon_vcage_v: 3178 case NEON::BI__builtin_neon_vcageq_v: 3179 case NEON::BI__builtin_neon_vcagt_v: 3180 case NEON::BI__builtin_neon_vcagtq_v: { 3181 llvm::Type *VecFlt = llvm::VectorType::get( 3182 VTy->getScalarSizeInBits() == 32 ? FloatTy : DoubleTy, 3183 VTy->getNumElements()); 3184 llvm::Type *Tys[] = { VTy, VecFlt }; 3185 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 3186 return EmitNeonCall(F, Ops, NameHint); 3187 } 3188 case NEON::BI__builtin_neon_vclz_v: 3189 case NEON::BI__builtin_neon_vclzq_v: 3190 // We generate target-independent intrinsic, which needs a second argument 3191 // for whether or not clz of zero is undefined; on ARM it isn't. 3192 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef())); 3193 break; 3194 case NEON::BI__builtin_neon_vcvt_f32_v: 3195 case NEON::BI__builtin_neon_vcvtq_f32_v: 3196 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3197 Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, Quad)); 3198 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") 3199 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); 3200 case NEON::BI__builtin_neon_vcvt_n_f32_v: 3201 case NEON::BI__builtin_neon_vcvt_n_f64_v: 3202 case NEON::BI__builtin_neon_vcvtq_n_f32_v: 3203 case NEON::BI__builtin_neon_vcvtq_n_f64_v: { 3204 llvm::Type *Tys[2] = { GetFloatNeonType(this, Type), Ty }; 3205 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic; 3206 Function *F = CGM.getIntrinsic(Int, Tys); 3207 return EmitNeonCall(F, Ops, "vcvt_n"); 3208 } 3209 case NEON::BI__builtin_neon_vcvt_n_s32_v: 3210 case NEON::BI__builtin_neon_vcvt_n_u32_v: 3211 case NEON::BI__builtin_neon_vcvt_n_s64_v: 3212 case NEON::BI__builtin_neon_vcvt_n_u64_v: 3213 case NEON::BI__builtin_neon_vcvtq_n_s32_v: 3214 case NEON::BI__builtin_neon_vcvtq_n_u32_v: 3215 case NEON::BI__builtin_neon_vcvtq_n_s64_v: 3216 case NEON::BI__builtin_neon_vcvtq_n_u64_v: { 3217 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 3218 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 3219 return EmitNeonCall(F, Ops, "vcvt_n"); 3220 } 3221 case NEON::BI__builtin_neon_vcvt_s32_v: 3222 case NEON::BI__builtin_neon_vcvt_u32_v: 3223 case NEON::BI__builtin_neon_vcvt_s64_v: 3224 case NEON::BI__builtin_neon_vcvt_u64_v: 3225 case NEON::BI__builtin_neon_vcvtq_s32_v: 3226 case NEON::BI__builtin_neon_vcvtq_u32_v: 3227 case NEON::BI__builtin_neon_vcvtq_s64_v: 3228 case NEON::BI__builtin_neon_vcvtq_u64_v: { 3229 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); 3230 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt") 3231 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt"); 3232 } 3233 case NEON::BI__builtin_neon_vcvta_s32_v: 3234 case NEON::BI__builtin_neon_vcvta_s64_v: 3235 case NEON::BI__builtin_neon_vcvta_u32_v: 3236 case NEON::BI__builtin_neon_vcvta_u64_v: 3237 case NEON::BI__builtin_neon_vcvtaq_s32_v: 3238 case NEON::BI__builtin_neon_vcvtaq_s64_v: 3239 case NEON::BI__builtin_neon_vcvtaq_u32_v: 3240 case NEON::BI__builtin_neon_vcvtaq_u64_v: 3241 case NEON::BI__builtin_neon_vcvtn_s32_v: 3242 case NEON::BI__builtin_neon_vcvtn_s64_v: 3243 case NEON::BI__builtin_neon_vcvtn_u32_v: 3244 case NEON::BI__builtin_neon_vcvtn_u64_v: 3245 case NEON::BI__builtin_neon_vcvtnq_s32_v: 3246 case NEON::BI__builtin_neon_vcvtnq_s64_v: 3247 case NEON::BI__builtin_neon_vcvtnq_u32_v: 3248 case NEON::BI__builtin_neon_vcvtnq_u64_v: 3249 case NEON::BI__builtin_neon_vcvtp_s32_v: 3250 case NEON::BI__builtin_neon_vcvtp_s64_v: 3251 case NEON::BI__builtin_neon_vcvtp_u32_v: 3252 case NEON::BI__builtin_neon_vcvtp_u64_v: 3253 case NEON::BI__builtin_neon_vcvtpq_s32_v: 3254 case NEON::BI__builtin_neon_vcvtpq_s64_v: 3255 case NEON::BI__builtin_neon_vcvtpq_u32_v: 3256 case NEON::BI__builtin_neon_vcvtpq_u64_v: 3257 case NEON::BI__builtin_neon_vcvtm_s32_v: 3258 case NEON::BI__builtin_neon_vcvtm_s64_v: 3259 case NEON::BI__builtin_neon_vcvtm_u32_v: 3260 case NEON::BI__builtin_neon_vcvtm_u64_v: 3261 case NEON::BI__builtin_neon_vcvtmq_s32_v: 3262 case NEON::BI__builtin_neon_vcvtmq_s64_v: 3263 case NEON::BI__builtin_neon_vcvtmq_u32_v: 3264 case NEON::BI__builtin_neon_vcvtmq_u64_v: { 3265 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 3266 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint); 3267 } 3268 case NEON::BI__builtin_neon_vext_v: 3269 case NEON::BI__builtin_neon_vextq_v: { 3270 int CV = cast<ConstantInt>(Ops[2])->getSExtValue(); 3271 SmallVector<Constant*, 16> Indices; 3272 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 3273 Indices.push_back(ConstantInt::get(Int32Ty, i+CV)); 3274 3275 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3276 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3277 Value *SV = llvm::ConstantVector::get(Indices); 3278 return Builder.CreateShuffleVector(Ops[0], Ops[1], SV, "vext"); 3279 } 3280 case NEON::BI__builtin_neon_vfma_v: 3281 case NEON::BI__builtin_neon_vfmaq_v: { 3282 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 3283 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3284 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3285 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 3286 3287 // NEON intrinsic puts accumulator first, unlike the LLVM fma. 3288 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); 3289 } 3290 case NEON::BI__builtin_neon_vld1_v: 3291 case NEON::BI__builtin_neon_vld1q_v: { 3292 llvm::Type *Tys[] = {Ty, Int8PtrTy}; 3293 Ops.push_back(getAlignmentValue32(PtrOp0)); 3294 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1"); 3295 } 3296 case NEON::BI__builtin_neon_vld2_v: 3297 case NEON::BI__builtin_neon_vld2q_v: 3298 case NEON::BI__builtin_neon_vld3_v: 3299 case NEON::BI__builtin_neon_vld3q_v: 3300 case NEON::BI__builtin_neon_vld4_v: 3301 case NEON::BI__builtin_neon_vld4q_v: { 3302 llvm::Type *Tys[] = {Ty, Int8PtrTy}; 3303 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 3304 Value *Align = getAlignmentValue32(PtrOp1); 3305 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint); 3306 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 3307 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3308 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 3309 } 3310 case NEON::BI__builtin_neon_vld1_dup_v: 3311 case NEON::BI__builtin_neon_vld1q_dup_v: { 3312 Value *V = UndefValue::get(Ty); 3313 Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 3314 PtrOp0 = Builder.CreateBitCast(PtrOp0, Ty); 3315 LoadInst *Ld = Builder.CreateLoad(PtrOp0); 3316 llvm::Constant *CI = ConstantInt::get(SizeTy, 0); 3317 Ops[0] = Builder.CreateInsertElement(V, Ld, CI); 3318 return EmitNeonSplat(Ops[0], CI); 3319 } 3320 case NEON::BI__builtin_neon_vld2_lane_v: 3321 case NEON::BI__builtin_neon_vld2q_lane_v: 3322 case NEON::BI__builtin_neon_vld3_lane_v: 3323 case NEON::BI__builtin_neon_vld3q_lane_v: 3324 case NEON::BI__builtin_neon_vld4_lane_v: 3325 case NEON::BI__builtin_neon_vld4q_lane_v: { 3326 llvm::Type *Tys[] = {Ty, Int8PtrTy}; 3327 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 3328 for (unsigned I = 2; I < Ops.size() - 1; ++I) 3329 Ops[I] = Builder.CreateBitCast(Ops[I], Ty); 3330 Ops.push_back(getAlignmentValue32(PtrOp1)); 3331 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint); 3332 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 3333 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3334 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 3335 } 3336 case NEON::BI__builtin_neon_vmovl_v: { 3337 llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy); 3338 Ops[0] = Builder.CreateBitCast(Ops[0], DTy); 3339 if (Usgn) 3340 return Builder.CreateZExt(Ops[0], Ty, "vmovl"); 3341 return Builder.CreateSExt(Ops[0], Ty, "vmovl"); 3342 } 3343 case NEON::BI__builtin_neon_vmovn_v: { 3344 llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy); 3345 Ops[0] = Builder.CreateBitCast(Ops[0], QTy); 3346 return Builder.CreateTrunc(Ops[0], Ty, "vmovn"); 3347 } 3348 case NEON::BI__builtin_neon_vmull_v: 3349 // FIXME: the integer vmull operations could be emitted in terms of pure 3350 // LLVM IR (2 exts followed by a mul). Unfortunately LLVM has a habit of 3351 // hoisting the exts outside loops. Until global ISel comes along that can 3352 // see through such movement this leads to bad CodeGen. So we need an 3353 // intrinsic for now. 3354 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls; 3355 Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int; 3356 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); 3357 case NEON::BI__builtin_neon_vpadal_v: 3358 case NEON::BI__builtin_neon_vpadalq_v: { 3359 // The source operand type has twice as many elements of half the size. 3360 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); 3361 llvm::Type *EltTy = 3362 llvm::IntegerType::get(getLLVMContext(), EltBits / 2); 3363 llvm::Type *NarrowTy = 3364 llvm::VectorType::get(EltTy, VTy->getNumElements() * 2); 3365 llvm::Type *Tys[2] = { Ty, NarrowTy }; 3366 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); 3367 } 3368 case NEON::BI__builtin_neon_vpaddl_v: 3369 case NEON::BI__builtin_neon_vpaddlq_v: { 3370 // The source operand type has twice as many elements of half the size. 3371 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); 3372 llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2); 3373 llvm::Type *NarrowTy = 3374 llvm::VectorType::get(EltTy, VTy->getNumElements() * 2); 3375 llvm::Type *Tys[2] = { Ty, NarrowTy }; 3376 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl"); 3377 } 3378 case NEON::BI__builtin_neon_vqdmlal_v: 3379 case NEON::BI__builtin_neon_vqdmlsl_v: { 3380 SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end()); 3381 Ops[1] = 3382 EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), MulOps, "vqdmlal"); 3383 Ops.resize(2); 3384 return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint); 3385 } 3386 case NEON::BI__builtin_neon_vqshl_n_v: 3387 case NEON::BI__builtin_neon_vqshlq_n_v: 3388 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n", 3389 1, false); 3390 case NEON::BI__builtin_neon_vqshlu_n_v: 3391 case NEON::BI__builtin_neon_vqshluq_n_v: 3392 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n", 3393 1, false); 3394 case NEON::BI__builtin_neon_vrecpe_v: 3395 case NEON::BI__builtin_neon_vrecpeq_v: 3396 case NEON::BI__builtin_neon_vrsqrte_v: 3397 case NEON::BI__builtin_neon_vrsqrteq_v: 3398 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic; 3399 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint); 3400 3401 case NEON::BI__builtin_neon_vrshr_n_v: 3402 case NEON::BI__builtin_neon_vrshrq_n_v: 3403 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", 3404 1, true); 3405 case NEON::BI__builtin_neon_vshl_n_v: 3406 case NEON::BI__builtin_neon_vshlq_n_v: 3407 Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false); 3408 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1], 3409 "vshl_n"); 3410 case NEON::BI__builtin_neon_vshll_n_v: { 3411 llvm::Type *SrcTy = llvm::VectorType::getTruncatedElementVectorType(VTy); 3412 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 3413 if (Usgn) 3414 Ops[0] = Builder.CreateZExt(Ops[0], VTy); 3415 else 3416 Ops[0] = Builder.CreateSExt(Ops[0], VTy); 3417 Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false); 3418 return Builder.CreateShl(Ops[0], Ops[1], "vshll_n"); 3419 } 3420 case NEON::BI__builtin_neon_vshrn_n_v: { 3421 llvm::Type *SrcTy = llvm::VectorType::getExtendedElementVectorType(VTy); 3422 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 3423 Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false); 3424 if (Usgn) 3425 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]); 3426 else 3427 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]); 3428 return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n"); 3429 } 3430 case NEON::BI__builtin_neon_vshr_n_v: 3431 case NEON::BI__builtin_neon_vshrq_n_v: 3432 return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n"); 3433 case NEON::BI__builtin_neon_vst1_v: 3434 case NEON::BI__builtin_neon_vst1q_v: 3435 case NEON::BI__builtin_neon_vst2_v: 3436 case NEON::BI__builtin_neon_vst2q_v: 3437 case NEON::BI__builtin_neon_vst3_v: 3438 case NEON::BI__builtin_neon_vst3q_v: 3439 case NEON::BI__builtin_neon_vst4_v: 3440 case NEON::BI__builtin_neon_vst4q_v: 3441 case NEON::BI__builtin_neon_vst2_lane_v: 3442 case NEON::BI__builtin_neon_vst2q_lane_v: 3443 case NEON::BI__builtin_neon_vst3_lane_v: 3444 case NEON::BI__builtin_neon_vst3q_lane_v: 3445 case NEON::BI__builtin_neon_vst4_lane_v: 3446 case NEON::BI__builtin_neon_vst4q_lane_v: { 3447 llvm::Type *Tys[] = {Int8PtrTy, Ty}; 3448 Ops.push_back(getAlignmentValue32(PtrOp0)); 3449 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, ""); 3450 } 3451 case NEON::BI__builtin_neon_vsubhn_v: { 3452 llvm::VectorType *SrcTy = 3453 llvm::VectorType::getExtendedElementVectorType(VTy); 3454 3455 // %sum = add <4 x i32> %lhs, %rhs 3456 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 3457 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); 3458 Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn"); 3459 3460 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16> 3461 Constant *ShiftAmt = 3462 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2); 3463 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn"); 3464 3465 // %res = trunc <4 x i32> %high to <4 x i16> 3466 return Builder.CreateTrunc(Ops[0], VTy, "vsubhn"); 3467 } 3468 case NEON::BI__builtin_neon_vtrn_v: 3469 case NEON::BI__builtin_neon_vtrnq_v: { 3470 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 3471 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3472 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 3473 Value *SV = nullptr; 3474 3475 for (unsigned vi = 0; vi != 2; ++vi) { 3476 SmallVector<Constant*, 16> Indices; 3477 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 3478 Indices.push_back(Builder.getInt32(i+vi)); 3479 Indices.push_back(Builder.getInt32(i+e+vi)); 3480 } 3481 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 3482 SV = llvm::ConstantVector::get(Indices); 3483 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn"); 3484 SV = Builder.CreateDefaultAlignedStore(SV, Addr); 3485 } 3486 return SV; 3487 } 3488 case NEON::BI__builtin_neon_vtst_v: 3489 case NEON::BI__builtin_neon_vtstq_v: { 3490 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3491 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3492 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); 3493 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], 3494 ConstantAggregateZero::get(Ty)); 3495 return Builder.CreateSExt(Ops[0], Ty, "vtst"); 3496 } 3497 case NEON::BI__builtin_neon_vuzp_v: 3498 case NEON::BI__builtin_neon_vuzpq_v: { 3499 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 3500 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3501 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 3502 Value *SV = nullptr; 3503 3504 for (unsigned vi = 0; vi != 2; ++vi) { 3505 SmallVector<Constant*, 16> Indices; 3506 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 3507 Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi)); 3508 3509 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 3510 SV = llvm::ConstantVector::get(Indices); 3511 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp"); 3512 SV = Builder.CreateDefaultAlignedStore(SV, Addr); 3513 } 3514 return SV; 3515 } 3516 case NEON::BI__builtin_neon_vzip_v: 3517 case NEON::BI__builtin_neon_vzipq_v: { 3518 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 3519 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3520 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 3521 Value *SV = nullptr; 3522 3523 for (unsigned vi = 0; vi != 2; ++vi) { 3524 SmallVector<Constant*, 16> Indices; 3525 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 3526 Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1)); 3527 Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e)); 3528 } 3529 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 3530 SV = llvm::ConstantVector::get(Indices); 3531 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip"); 3532 SV = Builder.CreateDefaultAlignedStore(SV, Addr); 3533 } 3534 return SV; 3535 } 3536 } 3537 3538 assert(Int && "Expected valid intrinsic number"); 3539 3540 // Determine the type(s) of this overloaded AArch64 intrinsic. 3541 Function *F = LookupNeonLLVMIntrinsic(Int, Modifier, Ty, E); 3542 3543 Value *Result = EmitNeonCall(F, Ops, NameHint); 3544 llvm::Type *ResultType = ConvertType(E->getType()); 3545 // AArch64 intrinsic one-element vector type cast to 3546 // scalar type expected by the builtin 3547 return Builder.CreateBitCast(Result, ResultType, NameHint); 3548 } 3549 3550 Value *CodeGenFunction::EmitAArch64CompareBuiltinExpr( 3551 Value *Op, llvm::Type *Ty, const CmpInst::Predicate Fp, 3552 const CmpInst::Predicate Ip, const Twine &Name) { 3553 llvm::Type *OTy = Op->getType(); 3554 3555 // FIXME: this is utterly horrific. We should not be looking at previous 3556 // codegen context to find out what needs doing. Unfortunately TableGen 3557 // currently gives us exactly the same calls for vceqz_f32 and vceqz_s32 3558 // (etc). 3559 if (BitCastInst *BI = dyn_cast<BitCastInst>(Op)) 3560 OTy = BI->getOperand(0)->getType(); 3561 3562 Op = Builder.CreateBitCast(Op, OTy); 3563 if (OTy->getScalarType()->isFloatingPointTy()) { 3564 Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy)); 3565 } else { 3566 Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy)); 3567 } 3568 return Builder.CreateSExt(Op, Ty, Name); 3569 } 3570 3571 static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops, 3572 Value *ExtOp, Value *IndexOp, 3573 llvm::Type *ResTy, unsigned IntID, 3574 const char *Name) { 3575 SmallVector<Value *, 2> TblOps; 3576 if (ExtOp) 3577 TblOps.push_back(ExtOp); 3578 3579 // Build a vector containing sequential number like (0, 1, 2, ..., 15) 3580 SmallVector<Constant*, 16> Indices; 3581 llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType()); 3582 for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) { 3583 Indices.push_back(ConstantInt::get(CGF.Int32Ty, 2*i)); 3584 Indices.push_back(ConstantInt::get(CGF.Int32Ty, 2*i+1)); 3585 } 3586 Value *SV = llvm::ConstantVector::get(Indices); 3587 3588 int PairPos = 0, End = Ops.size() - 1; 3589 while (PairPos < End) { 3590 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], 3591 Ops[PairPos+1], SV, Name)); 3592 PairPos += 2; 3593 } 3594 3595 // If there's an odd number of 64-bit lookup table, fill the high 64-bit 3596 // of the 128-bit lookup table with zero. 3597 if (PairPos == End) { 3598 Value *ZeroTbl = ConstantAggregateZero::get(TblTy); 3599 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], 3600 ZeroTbl, SV, Name)); 3601 } 3602 3603 Function *TblF; 3604 TblOps.push_back(IndexOp); 3605 TblF = CGF.CGM.getIntrinsic(IntID, ResTy); 3606 3607 return CGF.EmitNeonCall(TblF, TblOps, Name); 3608 } 3609 3610 Value *CodeGenFunction::GetValueForARMHint(unsigned BuiltinID) { 3611 unsigned Value; 3612 switch (BuiltinID) { 3613 default: 3614 return nullptr; 3615 case ARM::BI__builtin_arm_nop: 3616 Value = 0; 3617 break; 3618 case ARM::BI__builtin_arm_yield: 3619 case ARM::BI__yield: 3620 Value = 1; 3621 break; 3622 case ARM::BI__builtin_arm_wfe: 3623 case ARM::BI__wfe: 3624 Value = 2; 3625 break; 3626 case ARM::BI__builtin_arm_wfi: 3627 case ARM::BI__wfi: 3628 Value = 3; 3629 break; 3630 case ARM::BI__builtin_arm_sev: 3631 case ARM::BI__sev: 3632 Value = 4; 3633 break; 3634 case ARM::BI__builtin_arm_sevl: 3635 case ARM::BI__sevl: 3636 Value = 5; 3637 break; 3638 } 3639 3640 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint), 3641 llvm::ConstantInt::get(Int32Ty, Value)); 3642 } 3643 3644 // Generates the IR for the read/write special register builtin, 3645 // ValueType is the type of the value that is to be written or read, 3646 // RegisterType is the type of the register being written to or read from. 3647 static Value *EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, 3648 const CallExpr *E, 3649 llvm::Type *RegisterType, 3650 llvm::Type *ValueType, bool IsRead) { 3651 // write and register intrinsics only support 32 and 64 bit operations. 3652 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64)) 3653 && "Unsupported size for register."); 3654 3655 CodeGen::CGBuilderTy &Builder = CGF.Builder; 3656 CodeGen::CodeGenModule &CGM = CGF.CGM; 3657 LLVMContext &Context = CGM.getLLVMContext(); 3658 3659 const Expr *SysRegStrExpr = E->getArg(0)->IgnoreParenCasts(); 3660 StringRef SysReg = cast<StringLiteral>(SysRegStrExpr)->getString(); 3661 3662 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) }; 3663 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); 3664 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName); 3665 3666 llvm::Type *Types[] = { RegisterType }; 3667 3668 bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32); 3669 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64)) 3670 && "Can't fit 64-bit value in 32-bit register"); 3671 3672 if (IsRead) { 3673 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types); 3674 llvm::Value *Call = Builder.CreateCall(F, Metadata); 3675 3676 if (MixedTypes) 3677 // Read into 64 bit register and then truncate result to 32 bit. 3678 return Builder.CreateTrunc(Call, ValueType); 3679 3680 if (ValueType->isPointerTy()) 3681 // Have i32/i64 result (Call) but want to return a VoidPtrTy (i8*). 3682 return Builder.CreateIntToPtr(Call, ValueType); 3683 3684 return Call; 3685 } 3686 3687 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types); 3688 llvm::Value *ArgValue = CGF.EmitScalarExpr(E->getArg(1)); 3689 if (MixedTypes) { 3690 // Extend 32 bit write value to 64 bit to pass to write. 3691 ArgValue = Builder.CreateZExt(ArgValue, RegisterType); 3692 return Builder.CreateCall(F, { Metadata, ArgValue }); 3693 } 3694 3695 if (ValueType->isPointerTy()) { 3696 // Have VoidPtrTy ArgValue but want to return an i32/i64. 3697 ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType); 3698 return Builder.CreateCall(F, { Metadata, ArgValue }); 3699 } 3700 3701 return Builder.CreateCall(F, { Metadata, ArgValue }); 3702 } 3703 3704 /// Return true if BuiltinID is an overloaded Neon intrinsic with an extra 3705 /// argument that specifies the vector type. 3706 static bool HasExtraNeonArgument(unsigned BuiltinID) { 3707 switch (BuiltinID) { 3708 default: break; 3709 case NEON::BI__builtin_neon_vget_lane_i8: 3710 case NEON::BI__builtin_neon_vget_lane_i16: 3711 case NEON::BI__builtin_neon_vget_lane_i32: 3712 case NEON::BI__builtin_neon_vget_lane_i64: 3713 case NEON::BI__builtin_neon_vget_lane_f32: 3714 case NEON::BI__builtin_neon_vgetq_lane_i8: 3715 case NEON::BI__builtin_neon_vgetq_lane_i16: 3716 case NEON::BI__builtin_neon_vgetq_lane_i32: 3717 case NEON::BI__builtin_neon_vgetq_lane_i64: 3718 case NEON::BI__builtin_neon_vgetq_lane_f32: 3719 case NEON::BI__builtin_neon_vset_lane_i8: 3720 case NEON::BI__builtin_neon_vset_lane_i16: 3721 case NEON::BI__builtin_neon_vset_lane_i32: 3722 case NEON::BI__builtin_neon_vset_lane_i64: 3723 case NEON::BI__builtin_neon_vset_lane_f32: 3724 case NEON::BI__builtin_neon_vsetq_lane_i8: 3725 case NEON::BI__builtin_neon_vsetq_lane_i16: 3726 case NEON::BI__builtin_neon_vsetq_lane_i32: 3727 case NEON::BI__builtin_neon_vsetq_lane_i64: 3728 case NEON::BI__builtin_neon_vsetq_lane_f32: 3729 case NEON::BI__builtin_neon_vsha1h_u32: 3730 case NEON::BI__builtin_neon_vsha1cq_u32: 3731 case NEON::BI__builtin_neon_vsha1pq_u32: 3732 case NEON::BI__builtin_neon_vsha1mq_u32: 3733 case ARM::BI_MoveToCoprocessor: 3734 case ARM::BI_MoveToCoprocessor2: 3735 return false; 3736 } 3737 return true; 3738 } 3739 3740 Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, 3741 const CallExpr *E) { 3742 if (auto Hint = GetValueForARMHint(BuiltinID)) 3743 return Hint; 3744 3745 if (BuiltinID == ARM::BI__emit) { 3746 bool IsThumb = getTarget().getTriple().getArch() == llvm::Triple::thumb; 3747 llvm::FunctionType *FTy = 3748 llvm::FunctionType::get(VoidTy, /*Variadic=*/false); 3749 3750 APSInt Value; 3751 if (!E->getArg(0)->EvaluateAsInt(Value, CGM.getContext())) 3752 llvm_unreachable("Sema will ensure that the parameter is constant"); 3753 3754 uint64_t ZExtValue = Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue(); 3755 3756 llvm::InlineAsm *Emit = 3757 IsThumb ? InlineAsm::get(FTy, ".inst.n 0x" + utohexstr(ZExtValue), "", 3758 /*SideEffects=*/true) 3759 : InlineAsm::get(FTy, ".inst 0x" + utohexstr(ZExtValue), "", 3760 /*SideEffects=*/true); 3761 3762 return Builder.CreateCall(Emit); 3763 } 3764 3765 if (BuiltinID == ARM::BI__builtin_arm_dbg) { 3766 Value *Option = EmitScalarExpr(E->getArg(0)); 3767 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option); 3768 } 3769 3770 if (BuiltinID == ARM::BI__builtin_arm_prefetch) { 3771 Value *Address = EmitScalarExpr(E->getArg(0)); 3772 Value *RW = EmitScalarExpr(E->getArg(1)); 3773 Value *IsData = EmitScalarExpr(E->getArg(2)); 3774 3775 // Locality is not supported on ARM target 3776 Value *Locality = llvm::ConstantInt::get(Int32Ty, 3); 3777 3778 Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 3779 return Builder.CreateCall(F, {Address, RW, Locality, IsData}); 3780 } 3781 3782 if (BuiltinID == ARM::BI__builtin_arm_rbit) { 3783 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_rbit), 3784 EmitScalarExpr(E->getArg(0)), 3785 "rbit"); 3786 } 3787 3788 if (BuiltinID == ARM::BI__clear_cache) { 3789 assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments"); 3790 const FunctionDecl *FD = E->getDirectCallee(); 3791 Value *Ops[2]; 3792 for (unsigned i = 0; i < 2; i++) 3793 Ops[i] = EmitScalarExpr(E->getArg(i)); 3794 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType()); 3795 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty); 3796 StringRef Name = FD->getName(); 3797 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); 3798 } 3799 3800 if (BuiltinID == ARM::BI__builtin_arm_ldrexd || 3801 ((BuiltinID == ARM::BI__builtin_arm_ldrex || 3802 BuiltinID == ARM::BI__builtin_arm_ldaex) && 3803 getContext().getTypeSize(E->getType()) == 64) || 3804 BuiltinID == ARM::BI__ldrexd) { 3805 Function *F; 3806 3807 switch (BuiltinID) { 3808 default: llvm_unreachable("unexpected builtin"); 3809 case ARM::BI__builtin_arm_ldaex: 3810 F = CGM.getIntrinsic(Intrinsic::arm_ldaexd); 3811 break; 3812 case ARM::BI__builtin_arm_ldrexd: 3813 case ARM::BI__builtin_arm_ldrex: 3814 case ARM::BI__ldrexd: 3815 F = CGM.getIntrinsic(Intrinsic::arm_ldrexd); 3816 break; 3817 } 3818 3819 Value *LdPtr = EmitScalarExpr(E->getArg(0)); 3820 Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy), 3821 "ldrexd"); 3822 3823 Value *Val0 = Builder.CreateExtractValue(Val, 1); 3824 Value *Val1 = Builder.CreateExtractValue(Val, 0); 3825 Val0 = Builder.CreateZExt(Val0, Int64Ty); 3826 Val1 = Builder.CreateZExt(Val1, Int64Ty); 3827 3828 Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32); 3829 Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */); 3830 Val = Builder.CreateOr(Val, Val1); 3831 return Builder.CreateBitCast(Val, ConvertType(E->getType())); 3832 } 3833 3834 if (BuiltinID == ARM::BI__builtin_arm_ldrex || 3835 BuiltinID == ARM::BI__builtin_arm_ldaex) { 3836 Value *LoadAddr = EmitScalarExpr(E->getArg(0)); 3837 3838 QualType Ty = E->getType(); 3839 llvm::Type *RealResTy = ConvertType(Ty); 3840 llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(), 3841 getContext().getTypeSize(Ty)); 3842 LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo()); 3843 3844 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_ldaex 3845 ? Intrinsic::arm_ldaex 3846 : Intrinsic::arm_ldrex, 3847 LoadAddr->getType()); 3848 Value *Val = Builder.CreateCall(F, LoadAddr, "ldrex"); 3849 3850 if (RealResTy->isPointerTy()) 3851 return Builder.CreateIntToPtr(Val, RealResTy); 3852 else { 3853 Val = Builder.CreateTruncOrBitCast(Val, IntResTy); 3854 return Builder.CreateBitCast(Val, RealResTy); 3855 } 3856 } 3857 3858 if (BuiltinID == ARM::BI__builtin_arm_strexd || 3859 ((BuiltinID == ARM::BI__builtin_arm_stlex || 3860 BuiltinID == ARM::BI__builtin_arm_strex) && 3861 getContext().getTypeSize(E->getArg(0)->getType()) == 64)) { 3862 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex 3863 ? Intrinsic::arm_stlexd 3864 : Intrinsic::arm_strexd); 3865 llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, nullptr); 3866 3867 Address Tmp = CreateMemTemp(E->getArg(0)->getType()); 3868 Value *Val = EmitScalarExpr(E->getArg(0)); 3869 Builder.CreateStore(Val, Tmp); 3870 3871 Address LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy)); 3872 Val = Builder.CreateLoad(LdPtr); 3873 3874 Value *Arg0 = Builder.CreateExtractValue(Val, 0); 3875 Value *Arg1 = Builder.CreateExtractValue(Val, 1); 3876 Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), Int8PtrTy); 3877 return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "strexd"); 3878 } 3879 3880 if (BuiltinID == ARM::BI__builtin_arm_strex || 3881 BuiltinID == ARM::BI__builtin_arm_stlex) { 3882 Value *StoreVal = EmitScalarExpr(E->getArg(0)); 3883 Value *StoreAddr = EmitScalarExpr(E->getArg(1)); 3884 3885 QualType Ty = E->getArg(0)->getType(); 3886 llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(), 3887 getContext().getTypeSize(Ty)); 3888 StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo()); 3889 3890 if (StoreVal->getType()->isPointerTy()) 3891 StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty); 3892 else { 3893 StoreVal = Builder.CreateBitCast(StoreVal, StoreTy); 3894 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty); 3895 } 3896 3897 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex 3898 ? Intrinsic::arm_stlex 3899 : Intrinsic::arm_strex, 3900 StoreAddr->getType()); 3901 return Builder.CreateCall(F, {StoreVal, StoreAddr}, "strex"); 3902 } 3903 3904 if (BuiltinID == ARM::BI__builtin_arm_clrex) { 3905 Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex); 3906 return Builder.CreateCall(F); 3907 } 3908 3909 // CRC32 3910 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic; 3911 switch (BuiltinID) { 3912 case ARM::BI__builtin_arm_crc32b: 3913 CRCIntrinsicID = Intrinsic::arm_crc32b; break; 3914 case ARM::BI__builtin_arm_crc32cb: 3915 CRCIntrinsicID = Intrinsic::arm_crc32cb; break; 3916 case ARM::BI__builtin_arm_crc32h: 3917 CRCIntrinsicID = Intrinsic::arm_crc32h; break; 3918 case ARM::BI__builtin_arm_crc32ch: 3919 CRCIntrinsicID = Intrinsic::arm_crc32ch; break; 3920 case ARM::BI__builtin_arm_crc32w: 3921 case ARM::BI__builtin_arm_crc32d: 3922 CRCIntrinsicID = Intrinsic::arm_crc32w; break; 3923 case ARM::BI__builtin_arm_crc32cw: 3924 case ARM::BI__builtin_arm_crc32cd: 3925 CRCIntrinsicID = Intrinsic::arm_crc32cw; break; 3926 } 3927 3928 if (CRCIntrinsicID != Intrinsic::not_intrinsic) { 3929 Value *Arg0 = EmitScalarExpr(E->getArg(0)); 3930 Value *Arg1 = EmitScalarExpr(E->getArg(1)); 3931 3932 // crc32{c,}d intrinsics are implemnted as two calls to crc32{c,}w 3933 // intrinsics, hence we need different codegen for these cases. 3934 if (BuiltinID == ARM::BI__builtin_arm_crc32d || 3935 BuiltinID == ARM::BI__builtin_arm_crc32cd) { 3936 Value *C1 = llvm::ConstantInt::get(Int64Ty, 32); 3937 Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty); 3938 Value *Arg1b = Builder.CreateLShr(Arg1, C1); 3939 Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty); 3940 3941 Function *F = CGM.getIntrinsic(CRCIntrinsicID); 3942 Value *Res = Builder.CreateCall(F, {Arg0, Arg1a}); 3943 return Builder.CreateCall(F, {Res, Arg1b}); 3944 } else { 3945 Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty); 3946 3947 Function *F = CGM.getIntrinsic(CRCIntrinsicID); 3948 return Builder.CreateCall(F, {Arg0, Arg1}); 3949 } 3950 } 3951 3952 if (BuiltinID == ARM::BI__builtin_arm_rsr || 3953 BuiltinID == ARM::BI__builtin_arm_rsr64 || 3954 BuiltinID == ARM::BI__builtin_arm_rsrp || 3955 BuiltinID == ARM::BI__builtin_arm_wsr || 3956 BuiltinID == ARM::BI__builtin_arm_wsr64 || 3957 BuiltinID == ARM::BI__builtin_arm_wsrp) { 3958 3959 bool IsRead = BuiltinID == ARM::BI__builtin_arm_rsr || 3960 BuiltinID == ARM::BI__builtin_arm_rsr64 || 3961 BuiltinID == ARM::BI__builtin_arm_rsrp; 3962 3963 bool IsPointerBuiltin = BuiltinID == ARM::BI__builtin_arm_rsrp || 3964 BuiltinID == ARM::BI__builtin_arm_wsrp; 3965 3966 bool Is64Bit = BuiltinID == ARM::BI__builtin_arm_rsr64 || 3967 BuiltinID == ARM::BI__builtin_arm_wsr64; 3968 3969 llvm::Type *ValueType; 3970 llvm::Type *RegisterType; 3971 if (IsPointerBuiltin) { 3972 ValueType = VoidPtrTy; 3973 RegisterType = Int32Ty; 3974 } else if (Is64Bit) { 3975 ValueType = RegisterType = Int64Ty; 3976 } else { 3977 ValueType = RegisterType = Int32Ty; 3978 } 3979 3980 return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType, IsRead); 3981 } 3982 3983 // Find out if any arguments are required to be integer constant 3984 // expressions. 3985 unsigned ICEArguments = 0; 3986 ASTContext::GetBuiltinTypeError Error; 3987 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 3988 assert(Error == ASTContext::GE_None && "Should not codegen an error"); 3989 3990 auto getAlignmentValue32 = [&](Address addr) -> Value* { 3991 return Builder.getInt32(addr.getAlignment().getQuantity()); 3992 }; 3993 3994 Address PtrOp0 = Address::invalid(); 3995 Address PtrOp1 = Address::invalid(); 3996 SmallVector<Value*, 4> Ops; 3997 bool HasExtraArg = HasExtraNeonArgument(BuiltinID); 3998 unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0); 3999 for (unsigned i = 0, e = NumArgs; i != e; i++) { 4000 if (i == 0) { 4001 switch (BuiltinID) { 4002 case NEON::BI__builtin_neon_vld1_v: 4003 case NEON::BI__builtin_neon_vld1q_v: 4004 case NEON::BI__builtin_neon_vld1q_lane_v: 4005 case NEON::BI__builtin_neon_vld1_lane_v: 4006 case NEON::BI__builtin_neon_vld1_dup_v: 4007 case NEON::BI__builtin_neon_vld1q_dup_v: 4008 case NEON::BI__builtin_neon_vst1_v: 4009 case NEON::BI__builtin_neon_vst1q_v: 4010 case NEON::BI__builtin_neon_vst1q_lane_v: 4011 case NEON::BI__builtin_neon_vst1_lane_v: 4012 case NEON::BI__builtin_neon_vst2_v: 4013 case NEON::BI__builtin_neon_vst2q_v: 4014 case NEON::BI__builtin_neon_vst2_lane_v: 4015 case NEON::BI__builtin_neon_vst2q_lane_v: 4016 case NEON::BI__builtin_neon_vst3_v: 4017 case NEON::BI__builtin_neon_vst3q_v: 4018 case NEON::BI__builtin_neon_vst3_lane_v: 4019 case NEON::BI__builtin_neon_vst3q_lane_v: 4020 case NEON::BI__builtin_neon_vst4_v: 4021 case NEON::BI__builtin_neon_vst4q_v: 4022 case NEON::BI__builtin_neon_vst4_lane_v: 4023 case NEON::BI__builtin_neon_vst4q_lane_v: 4024 // Get the alignment for the argument in addition to the value; 4025 // we'll use it later. 4026 PtrOp0 = EmitPointerWithAlignment(E->getArg(0)); 4027 Ops.push_back(PtrOp0.getPointer()); 4028 continue; 4029 } 4030 } 4031 if (i == 1) { 4032 switch (BuiltinID) { 4033 case NEON::BI__builtin_neon_vld2_v: 4034 case NEON::BI__builtin_neon_vld2q_v: 4035 case NEON::BI__builtin_neon_vld3_v: 4036 case NEON::BI__builtin_neon_vld3q_v: 4037 case NEON::BI__builtin_neon_vld4_v: 4038 case NEON::BI__builtin_neon_vld4q_v: 4039 case NEON::BI__builtin_neon_vld2_lane_v: 4040 case NEON::BI__builtin_neon_vld2q_lane_v: 4041 case NEON::BI__builtin_neon_vld3_lane_v: 4042 case NEON::BI__builtin_neon_vld3q_lane_v: 4043 case NEON::BI__builtin_neon_vld4_lane_v: 4044 case NEON::BI__builtin_neon_vld4q_lane_v: 4045 case NEON::BI__builtin_neon_vld2_dup_v: 4046 case NEON::BI__builtin_neon_vld3_dup_v: 4047 case NEON::BI__builtin_neon_vld4_dup_v: 4048 // Get the alignment for the argument in addition to the value; 4049 // we'll use it later. 4050 PtrOp1 = EmitPointerWithAlignment(E->getArg(1)); 4051 Ops.push_back(PtrOp1.getPointer()); 4052 continue; 4053 } 4054 } 4055 4056 if ((ICEArguments & (1 << i)) == 0) { 4057 Ops.push_back(EmitScalarExpr(E->getArg(i))); 4058 } else { 4059 // If this is required to be a constant, constant fold it so that we know 4060 // that the generated intrinsic gets a ConstantInt. 4061 llvm::APSInt Result; 4062 bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext()); 4063 assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst; 4064 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); 4065 } 4066 } 4067 4068 switch (BuiltinID) { 4069 default: break; 4070 4071 case NEON::BI__builtin_neon_vget_lane_i8: 4072 case NEON::BI__builtin_neon_vget_lane_i16: 4073 case NEON::BI__builtin_neon_vget_lane_i32: 4074 case NEON::BI__builtin_neon_vget_lane_i64: 4075 case NEON::BI__builtin_neon_vget_lane_f32: 4076 case NEON::BI__builtin_neon_vgetq_lane_i8: 4077 case NEON::BI__builtin_neon_vgetq_lane_i16: 4078 case NEON::BI__builtin_neon_vgetq_lane_i32: 4079 case NEON::BI__builtin_neon_vgetq_lane_i64: 4080 case NEON::BI__builtin_neon_vgetq_lane_f32: 4081 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane"); 4082 4083 case NEON::BI__builtin_neon_vset_lane_i8: 4084 case NEON::BI__builtin_neon_vset_lane_i16: 4085 case NEON::BI__builtin_neon_vset_lane_i32: 4086 case NEON::BI__builtin_neon_vset_lane_i64: 4087 case NEON::BI__builtin_neon_vset_lane_f32: 4088 case NEON::BI__builtin_neon_vsetq_lane_i8: 4089 case NEON::BI__builtin_neon_vsetq_lane_i16: 4090 case NEON::BI__builtin_neon_vsetq_lane_i32: 4091 case NEON::BI__builtin_neon_vsetq_lane_i64: 4092 case NEON::BI__builtin_neon_vsetq_lane_f32: 4093 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 4094 4095 case NEON::BI__builtin_neon_vsha1h_u32: 4096 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops, 4097 "vsha1h"); 4098 case NEON::BI__builtin_neon_vsha1cq_u32: 4099 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops, 4100 "vsha1h"); 4101 case NEON::BI__builtin_neon_vsha1pq_u32: 4102 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops, 4103 "vsha1h"); 4104 case NEON::BI__builtin_neon_vsha1mq_u32: 4105 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops, 4106 "vsha1h"); 4107 4108 // The ARM _MoveToCoprocessor builtins put the input register value as 4109 // the first argument, but the LLVM intrinsic expects it as the third one. 4110 case ARM::BI_MoveToCoprocessor: 4111 case ARM::BI_MoveToCoprocessor2: { 4112 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI_MoveToCoprocessor ? 4113 Intrinsic::arm_mcr : Intrinsic::arm_mcr2); 4114 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0], 4115 Ops[3], Ops[4], Ops[5]}); 4116 } 4117 } 4118 4119 // Get the last argument, which specifies the vector type. 4120 assert(HasExtraArg); 4121 llvm::APSInt Result; 4122 const Expr *Arg = E->getArg(E->getNumArgs()-1); 4123 if (!Arg->isIntegerConstantExpr(Result, getContext())) 4124 return nullptr; 4125 4126 if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f || 4127 BuiltinID == ARM::BI__builtin_arm_vcvtr_d) { 4128 // Determine the overloaded type of this builtin. 4129 llvm::Type *Ty; 4130 if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f) 4131 Ty = FloatTy; 4132 else 4133 Ty = DoubleTy; 4134 4135 // Determine whether this is an unsigned conversion or not. 4136 bool usgn = Result.getZExtValue() == 1; 4137 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr; 4138 4139 // Call the appropriate intrinsic. 4140 Function *F = CGM.getIntrinsic(Int, Ty); 4141 return Builder.CreateCall(F, Ops, "vcvtr"); 4142 } 4143 4144 // Determine the type of this overloaded NEON intrinsic. 4145 NeonTypeFlags Type(Result.getZExtValue()); 4146 bool usgn = Type.isUnsigned(); 4147 bool rightShift = false; 4148 4149 llvm::VectorType *VTy = GetNeonType(this, Type); 4150 llvm::Type *Ty = VTy; 4151 if (!Ty) 4152 return nullptr; 4153 4154 // Many NEON builtins have identical semantics and uses in ARM and 4155 // AArch64. Emit these in a single function. 4156 auto IntrinsicMap = makeArrayRef(ARMSIMDIntrinsicMap); 4157 const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap( 4158 IntrinsicMap, BuiltinID, NEONSIMDIntrinsicsProvenSorted); 4159 if (Builtin) 4160 return EmitCommonNeonBuiltinExpr( 4161 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic, 4162 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1); 4163 4164 unsigned Int; 4165 switch (BuiltinID) { 4166 default: return nullptr; 4167 case NEON::BI__builtin_neon_vld1q_lane_v: 4168 // Handle 64-bit integer elements as a special case. Use shuffles of 4169 // one-element vectors to avoid poor code for i64 in the backend. 4170 if (VTy->getElementType()->isIntegerTy(64)) { 4171 // Extract the other lane. 4172 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4173 uint32_t Lane = cast<ConstantInt>(Ops[2])->getZExtValue(); 4174 Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane)); 4175 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); 4176 // Load the value as a one-element vector. 4177 Ty = llvm::VectorType::get(VTy->getElementType(), 1); 4178 llvm::Type *Tys[] = {Ty, Int8PtrTy}; 4179 Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Tys); 4180 Value *Align = getAlignmentValue32(PtrOp0); 4181 Value *Ld = Builder.CreateCall(F, {Ops[0], Align}); 4182 // Combine them. 4183 uint32_t Indices[] = {1 - Lane, Lane}; 4184 SV = llvm::ConstantDataVector::get(getLLVMContext(), Indices); 4185 return Builder.CreateShuffleVector(Ops[1], Ld, SV, "vld1q_lane"); 4186 } 4187 // fall through 4188 case NEON::BI__builtin_neon_vld1_lane_v: { 4189 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4190 PtrOp0 = Builder.CreateElementBitCast(PtrOp0, VTy->getElementType()); 4191 Value *Ld = Builder.CreateLoad(PtrOp0); 4192 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane"); 4193 } 4194 case NEON::BI__builtin_neon_vld2_dup_v: 4195 case NEON::BI__builtin_neon_vld3_dup_v: 4196 case NEON::BI__builtin_neon_vld4_dup_v: { 4197 // Handle 64-bit elements as a special-case. There is no "dup" needed. 4198 if (VTy->getElementType()->getPrimitiveSizeInBits() == 64) { 4199 switch (BuiltinID) { 4200 case NEON::BI__builtin_neon_vld2_dup_v: 4201 Int = Intrinsic::arm_neon_vld2; 4202 break; 4203 case NEON::BI__builtin_neon_vld3_dup_v: 4204 Int = Intrinsic::arm_neon_vld3; 4205 break; 4206 case NEON::BI__builtin_neon_vld4_dup_v: 4207 Int = Intrinsic::arm_neon_vld4; 4208 break; 4209 default: llvm_unreachable("unknown vld_dup intrinsic?"); 4210 } 4211 llvm::Type *Tys[] = {Ty, Int8PtrTy}; 4212 Function *F = CGM.getIntrinsic(Int, Tys); 4213 llvm::Value *Align = getAlignmentValue32(PtrOp1); 4214 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, "vld_dup"); 4215 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 4216 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4217 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 4218 } 4219 switch (BuiltinID) { 4220 case NEON::BI__builtin_neon_vld2_dup_v: 4221 Int = Intrinsic::arm_neon_vld2lane; 4222 break; 4223 case NEON::BI__builtin_neon_vld3_dup_v: 4224 Int = Intrinsic::arm_neon_vld3lane; 4225 break; 4226 case NEON::BI__builtin_neon_vld4_dup_v: 4227 Int = Intrinsic::arm_neon_vld4lane; 4228 break; 4229 default: llvm_unreachable("unknown vld_dup intrinsic?"); 4230 } 4231 llvm::Type *Tys[] = {Ty, Int8PtrTy}; 4232 Function *F = CGM.getIntrinsic(Int, Tys); 4233 llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType()); 4234 4235 SmallVector<Value*, 6> Args; 4236 Args.push_back(Ops[1]); 4237 Args.append(STy->getNumElements(), UndefValue::get(Ty)); 4238 4239 llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); 4240 Args.push_back(CI); 4241 Args.push_back(getAlignmentValue32(PtrOp1)); 4242 4243 Ops[1] = Builder.CreateCall(F, Args, "vld_dup"); 4244 // splat lane 0 to all elts in each vector of the result. 4245 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { 4246 Value *Val = Builder.CreateExtractValue(Ops[1], i); 4247 Value *Elt = Builder.CreateBitCast(Val, Ty); 4248 Elt = EmitNeonSplat(Elt, CI); 4249 Elt = Builder.CreateBitCast(Elt, Val->getType()); 4250 Ops[1] = Builder.CreateInsertValue(Ops[1], Elt, i); 4251 } 4252 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 4253 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4254 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 4255 } 4256 case NEON::BI__builtin_neon_vqrshrn_n_v: 4257 Int = 4258 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns; 4259 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n", 4260 1, true); 4261 case NEON::BI__builtin_neon_vqrshrun_n_v: 4262 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty), 4263 Ops, "vqrshrun_n", 1, true); 4264 case NEON::BI__builtin_neon_vqshrn_n_v: 4265 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns; 4266 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n", 4267 1, true); 4268 case NEON::BI__builtin_neon_vqshrun_n_v: 4269 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty), 4270 Ops, "vqshrun_n", 1, true); 4271 case NEON::BI__builtin_neon_vrecpe_v: 4272 case NEON::BI__builtin_neon_vrecpeq_v: 4273 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty), 4274 Ops, "vrecpe"); 4275 case NEON::BI__builtin_neon_vrshrn_n_v: 4276 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty), 4277 Ops, "vrshrn_n", 1, true); 4278 case NEON::BI__builtin_neon_vrsra_n_v: 4279 case NEON::BI__builtin_neon_vrsraq_n_v: 4280 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4281 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4282 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true); 4283 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts; 4284 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]}); 4285 return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n"); 4286 case NEON::BI__builtin_neon_vsri_n_v: 4287 case NEON::BI__builtin_neon_vsriq_n_v: 4288 rightShift = true; 4289 case NEON::BI__builtin_neon_vsli_n_v: 4290 case NEON::BI__builtin_neon_vsliq_n_v: 4291 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift); 4292 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty), 4293 Ops, "vsli_n"); 4294 case NEON::BI__builtin_neon_vsra_n_v: 4295 case NEON::BI__builtin_neon_vsraq_n_v: 4296 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4297 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); 4298 return Builder.CreateAdd(Ops[0], Ops[1]); 4299 case NEON::BI__builtin_neon_vst1q_lane_v: 4300 // Handle 64-bit integer elements as a special case. Use a shuffle to get 4301 // a one-element vector and avoid poor code for i64 in the backend. 4302 if (VTy->getElementType()->isIntegerTy(64)) { 4303 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4304 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2])); 4305 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); 4306 Ops[2] = getAlignmentValue32(PtrOp0); 4307 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()}; 4308 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1, 4309 Tys), Ops); 4310 } 4311 // fall through 4312 case NEON::BI__builtin_neon_vst1_lane_v: { 4313 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4314 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); 4315 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 4316 auto St = Builder.CreateStore(Ops[1], Builder.CreateBitCast(PtrOp0, Ty)); 4317 return St; 4318 } 4319 case NEON::BI__builtin_neon_vtbl1_v: 4320 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1), 4321 Ops, "vtbl1"); 4322 case NEON::BI__builtin_neon_vtbl2_v: 4323 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2), 4324 Ops, "vtbl2"); 4325 case NEON::BI__builtin_neon_vtbl3_v: 4326 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3), 4327 Ops, "vtbl3"); 4328 case NEON::BI__builtin_neon_vtbl4_v: 4329 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4), 4330 Ops, "vtbl4"); 4331 case NEON::BI__builtin_neon_vtbx1_v: 4332 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1), 4333 Ops, "vtbx1"); 4334 case NEON::BI__builtin_neon_vtbx2_v: 4335 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2), 4336 Ops, "vtbx2"); 4337 case NEON::BI__builtin_neon_vtbx3_v: 4338 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3), 4339 Ops, "vtbx3"); 4340 case NEON::BI__builtin_neon_vtbx4_v: 4341 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4), 4342 Ops, "vtbx4"); 4343 } 4344 } 4345 4346 static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, 4347 const CallExpr *E, 4348 SmallVectorImpl<Value *> &Ops) { 4349 unsigned int Int = 0; 4350 const char *s = nullptr; 4351 4352 switch (BuiltinID) { 4353 default: 4354 return nullptr; 4355 case NEON::BI__builtin_neon_vtbl1_v: 4356 case NEON::BI__builtin_neon_vqtbl1_v: 4357 case NEON::BI__builtin_neon_vqtbl1q_v: 4358 case NEON::BI__builtin_neon_vtbl2_v: 4359 case NEON::BI__builtin_neon_vqtbl2_v: 4360 case NEON::BI__builtin_neon_vqtbl2q_v: 4361 case NEON::BI__builtin_neon_vtbl3_v: 4362 case NEON::BI__builtin_neon_vqtbl3_v: 4363 case NEON::BI__builtin_neon_vqtbl3q_v: 4364 case NEON::BI__builtin_neon_vtbl4_v: 4365 case NEON::BI__builtin_neon_vqtbl4_v: 4366 case NEON::BI__builtin_neon_vqtbl4q_v: 4367 break; 4368 case NEON::BI__builtin_neon_vtbx1_v: 4369 case NEON::BI__builtin_neon_vqtbx1_v: 4370 case NEON::BI__builtin_neon_vqtbx1q_v: 4371 case NEON::BI__builtin_neon_vtbx2_v: 4372 case NEON::BI__builtin_neon_vqtbx2_v: 4373 case NEON::BI__builtin_neon_vqtbx2q_v: 4374 case NEON::BI__builtin_neon_vtbx3_v: 4375 case NEON::BI__builtin_neon_vqtbx3_v: 4376 case NEON::BI__builtin_neon_vqtbx3q_v: 4377 case NEON::BI__builtin_neon_vtbx4_v: 4378 case NEON::BI__builtin_neon_vqtbx4_v: 4379 case NEON::BI__builtin_neon_vqtbx4q_v: 4380 break; 4381 } 4382 4383 assert(E->getNumArgs() >= 3); 4384 4385 // Get the last argument, which specifies the vector type. 4386 llvm::APSInt Result; 4387 const Expr *Arg = E->getArg(E->getNumArgs() - 1); 4388 if (!Arg->isIntegerConstantExpr(Result, CGF.getContext())) 4389 return nullptr; 4390 4391 // Determine the type of this overloaded NEON intrinsic. 4392 NeonTypeFlags Type(Result.getZExtValue()); 4393 llvm::VectorType *Ty = GetNeonType(&CGF, Type); 4394 if (!Ty) 4395 return nullptr; 4396 4397 CodeGen::CGBuilderTy &Builder = CGF.Builder; 4398 4399 // AArch64 scalar builtins are not overloaded, they do not have an extra 4400 // argument that specifies the vector type, need to handle each case. 4401 switch (BuiltinID) { 4402 case NEON::BI__builtin_neon_vtbl1_v: { 4403 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 1), nullptr, 4404 Ops[1], Ty, Intrinsic::aarch64_neon_tbl1, 4405 "vtbl1"); 4406 } 4407 case NEON::BI__builtin_neon_vtbl2_v: { 4408 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 2), nullptr, 4409 Ops[2], Ty, Intrinsic::aarch64_neon_tbl1, 4410 "vtbl1"); 4411 } 4412 case NEON::BI__builtin_neon_vtbl3_v: { 4413 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 3), nullptr, 4414 Ops[3], Ty, Intrinsic::aarch64_neon_tbl2, 4415 "vtbl2"); 4416 } 4417 case NEON::BI__builtin_neon_vtbl4_v: { 4418 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 4), nullptr, 4419 Ops[4], Ty, Intrinsic::aarch64_neon_tbl2, 4420 "vtbl2"); 4421 } 4422 case NEON::BI__builtin_neon_vtbx1_v: { 4423 Value *TblRes = 4424 packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 1), nullptr, Ops[2], 4425 Ty, Intrinsic::aarch64_neon_tbl1, "vtbl1"); 4426 4427 llvm::Constant *EightV = ConstantInt::get(Ty, 8); 4428 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV); 4429 CmpRes = Builder.CreateSExt(CmpRes, Ty); 4430 4431 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); 4432 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes); 4433 return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx"); 4434 } 4435 case NEON::BI__builtin_neon_vtbx2_v: { 4436 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 2), Ops[0], 4437 Ops[3], Ty, Intrinsic::aarch64_neon_tbx1, 4438 "vtbx1"); 4439 } 4440 case NEON::BI__builtin_neon_vtbx3_v: { 4441 Value *TblRes = 4442 packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 3), nullptr, Ops[4], 4443 Ty, Intrinsic::aarch64_neon_tbl2, "vtbl2"); 4444 4445 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24); 4446 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4], 4447 TwentyFourV); 4448 CmpRes = Builder.CreateSExt(CmpRes, Ty); 4449 4450 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); 4451 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes); 4452 return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx"); 4453 } 4454 case NEON::BI__builtin_neon_vtbx4_v: { 4455 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 4), Ops[0], 4456 Ops[5], Ty, Intrinsic::aarch64_neon_tbx2, 4457 "vtbx2"); 4458 } 4459 case NEON::BI__builtin_neon_vqtbl1_v: 4460 case NEON::BI__builtin_neon_vqtbl1q_v: 4461 Int = Intrinsic::aarch64_neon_tbl1; s = "vtbl1"; break; 4462 case NEON::BI__builtin_neon_vqtbl2_v: 4463 case NEON::BI__builtin_neon_vqtbl2q_v: { 4464 Int = Intrinsic::aarch64_neon_tbl2; s = "vtbl2"; break; 4465 case NEON::BI__builtin_neon_vqtbl3_v: 4466 case NEON::BI__builtin_neon_vqtbl3q_v: 4467 Int = Intrinsic::aarch64_neon_tbl3; s = "vtbl3"; break; 4468 case NEON::BI__builtin_neon_vqtbl4_v: 4469 case NEON::BI__builtin_neon_vqtbl4q_v: 4470 Int = Intrinsic::aarch64_neon_tbl4; s = "vtbl4"; break; 4471 case NEON::BI__builtin_neon_vqtbx1_v: 4472 case NEON::BI__builtin_neon_vqtbx1q_v: 4473 Int = Intrinsic::aarch64_neon_tbx1; s = "vtbx1"; break; 4474 case NEON::BI__builtin_neon_vqtbx2_v: 4475 case NEON::BI__builtin_neon_vqtbx2q_v: 4476 Int = Intrinsic::aarch64_neon_tbx2; s = "vtbx2"; break; 4477 case NEON::BI__builtin_neon_vqtbx3_v: 4478 case NEON::BI__builtin_neon_vqtbx3q_v: 4479 Int = Intrinsic::aarch64_neon_tbx3; s = "vtbx3"; break; 4480 case NEON::BI__builtin_neon_vqtbx4_v: 4481 case NEON::BI__builtin_neon_vqtbx4q_v: 4482 Int = Intrinsic::aarch64_neon_tbx4; s = "vtbx4"; break; 4483 } 4484 } 4485 4486 if (!Int) 4487 return nullptr; 4488 4489 Function *F = CGF.CGM.getIntrinsic(Int, Ty); 4490 return CGF.EmitNeonCall(F, Ops, s); 4491 } 4492 4493 Value *CodeGenFunction::vectorWrapScalar16(Value *Op) { 4494 llvm::Type *VTy = llvm::VectorType::get(Int16Ty, 4); 4495 Op = Builder.CreateBitCast(Op, Int16Ty); 4496 Value *V = UndefValue::get(VTy); 4497 llvm::Constant *CI = ConstantInt::get(SizeTy, 0); 4498 Op = Builder.CreateInsertElement(V, Op, CI); 4499 return Op; 4500 } 4501 4502 Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, 4503 const CallExpr *E) { 4504 unsigned HintID = static_cast<unsigned>(-1); 4505 switch (BuiltinID) { 4506 default: break; 4507 case AArch64::BI__builtin_arm_nop: 4508 HintID = 0; 4509 break; 4510 case AArch64::BI__builtin_arm_yield: 4511 HintID = 1; 4512 break; 4513 case AArch64::BI__builtin_arm_wfe: 4514 HintID = 2; 4515 break; 4516 case AArch64::BI__builtin_arm_wfi: 4517 HintID = 3; 4518 break; 4519 case AArch64::BI__builtin_arm_sev: 4520 HintID = 4; 4521 break; 4522 case AArch64::BI__builtin_arm_sevl: 4523 HintID = 5; 4524 break; 4525 } 4526 4527 if (HintID != static_cast<unsigned>(-1)) { 4528 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hint); 4529 return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID)); 4530 } 4531 4532 if (BuiltinID == AArch64::BI__builtin_arm_prefetch) { 4533 Value *Address = EmitScalarExpr(E->getArg(0)); 4534 Value *RW = EmitScalarExpr(E->getArg(1)); 4535 Value *CacheLevel = EmitScalarExpr(E->getArg(2)); 4536 Value *RetentionPolicy = EmitScalarExpr(E->getArg(3)); 4537 Value *IsData = EmitScalarExpr(E->getArg(4)); 4538 4539 Value *Locality = nullptr; 4540 if (cast<llvm::ConstantInt>(RetentionPolicy)->isZero()) { 4541 // Temporal fetch, needs to convert cache level to locality. 4542 Locality = llvm::ConstantInt::get(Int32Ty, 4543 -cast<llvm::ConstantInt>(CacheLevel)->getValue() + 3); 4544 } else { 4545 // Streaming fetch. 4546 Locality = llvm::ConstantInt::get(Int32Ty, 0); 4547 } 4548 4549 // FIXME: We need AArch64 specific LLVM intrinsic if we want to specify 4550 // PLDL3STRM or PLDL2STRM. 4551 Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 4552 return Builder.CreateCall(F, {Address, RW, Locality, IsData}); 4553 } 4554 4555 if (BuiltinID == AArch64::BI__builtin_arm_rbit) { 4556 assert((getContext().getTypeSize(E->getType()) == 32) && 4557 "rbit of unusual size!"); 4558 llvm::Value *Arg = EmitScalarExpr(E->getArg(0)); 4559 return Builder.CreateCall( 4560 CGM.getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg, "rbit"); 4561 } 4562 if (BuiltinID == AArch64::BI__builtin_arm_rbit64) { 4563 assert((getContext().getTypeSize(E->getType()) == 64) && 4564 "rbit of unusual size!"); 4565 llvm::Value *Arg = EmitScalarExpr(E->getArg(0)); 4566 return Builder.CreateCall( 4567 CGM.getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg, "rbit"); 4568 } 4569 4570 if (BuiltinID == AArch64::BI__clear_cache) { 4571 assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments"); 4572 const FunctionDecl *FD = E->getDirectCallee(); 4573 Value *Ops[2]; 4574 for (unsigned i = 0; i < 2; i++) 4575 Ops[i] = EmitScalarExpr(E->getArg(i)); 4576 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType()); 4577 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty); 4578 StringRef Name = FD->getName(); 4579 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); 4580 } 4581 4582 if ((BuiltinID == AArch64::BI__builtin_arm_ldrex || 4583 BuiltinID == AArch64::BI__builtin_arm_ldaex) && 4584 getContext().getTypeSize(E->getType()) == 128) { 4585 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex 4586 ? Intrinsic::aarch64_ldaxp 4587 : Intrinsic::aarch64_ldxp); 4588 4589 Value *LdPtr = EmitScalarExpr(E->getArg(0)); 4590 Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy), 4591 "ldxp"); 4592 4593 Value *Val0 = Builder.CreateExtractValue(Val, 1); 4594 Value *Val1 = Builder.CreateExtractValue(Val, 0); 4595 llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128); 4596 Val0 = Builder.CreateZExt(Val0, Int128Ty); 4597 Val1 = Builder.CreateZExt(Val1, Int128Ty); 4598 4599 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64); 4600 Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */); 4601 Val = Builder.CreateOr(Val, Val1); 4602 return Builder.CreateBitCast(Val, ConvertType(E->getType())); 4603 } else if (BuiltinID == AArch64::BI__builtin_arm_ldrex || 4604 BuiltinID == AArch64::BI__builtin_arm_ldaex) { 4605 Value *LoadAddr = EmitScalarExpr(E->getArg(0)); 4606 4607 QualType Ty = E->getType(); 4608 llvm::Type *RealResTy = ConvertType(Ty); 4609 llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(), 4610 getContext().getTypeSize(Ty)); 4611 LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo()); 4612 4613 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex 4614 ? Intrinsic::aarch64_ldaxr 4615 : Intrinsic::aarch64_ldxr, 4616 LoadAddr->getType()); 4617 Value *Val = Builder.CreateCall(F, LoadAddr, "ldxr"); 4618 4619 if (RealResTy->isPointerTy()) 4620 return Builder.CreateIntToPtr(Val, RealResTy); 4621 4622 Val = Builder.CreateTruncOrBitCast(Val, IntResTy); 4623 return Builder.CreateBitCast(Val, RealResTy); 4624 } 4625 4626 if ((BuiltinID == AArch64::BI__builtin_arm_strex || 4627 BuiltinID == AArch64::BI__builtin_arm_stlex) && 4628 getContext().getTypeSize(E->getArg(0)->getType()) == 128) { 4629 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex 4630 ? Intrinsic::aarch64_stlxp 4631 : Intrinsic::aarch64_stxp); 4632 llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty, nullptr); 4633 4634 Address Tmp = CreateMemTemp(E->getArg(0)->getType()); 4635 EmitAnyExprToMem(E->getArg(0), Tmp, Qualifiers(), /*init*/ true); 4636 4637 Tmp = Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(STy)); 4638 llvm::Value *Val = Builder.CreateLoad(Tmp); 4639 4640 Value *Arg0 = Builder.CreateExtractValue(Val, 0); 4641 Value *Arg1 = Builder.CreateExtractValue(Val, 1); 4642 Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), 4643 Int8PtrTy); 4644 return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "stxp"); 4645 } 4646 4647 if (BuiltinID == AArch64::BI__builtin_arm_strex || 4648 BuiltinID == AArch64::BI__builtin_arm_stlex) { 4649 Value *StoreVal = EmitScalarExpr(E->getArg(0)); 4650 Value *StoreAddr = EmitScalarExpr(E->getArg(1)); 4651 4652 QualType Ty = E->getArg(0)->getType(); 4653 llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(), 4654 getContext().getTypeSize(Ty)); 4655 StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo()); 4656 4657 if (StoreVal->getType()->isPointerTy()) 4658 StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty); 4659 else { 4660 StoreVal = Builder.CreateBitCast(StoreVal, StoreTy); 4661 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty); 4662 } 4663 4664 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex 4665 ? Intrinsic::aarch64_stlxr 4666 : Intrinsic::aarch64_stxr, 4667 StoreAddr->getType()); 4668 return Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr"); 4669 } 4670 4671 if (BuiltinID == AArch64::BI__builtin_arm_clrex) { 4672 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex); 4673 return Builder.CreateCall(F); 4674 } 4675 4676 // CRC32 4677 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic; 4678 switch (BuiltinID) { 4679 case AArch64::BI__builtin_arm_crc32b: 4680 CRCIntrinsicID = Intrinsic::aarch64_crc32b; break; 4681 case AArch64::BI__builtin_arm_crc32cb: 4682 CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break; 4683 case AArch64::BI__builtin_arm_crc32h: 4684 CRCIntrinsicID = Intrinsic::aarch64_crc32h; break; 4685 case AArch64::BI__builtin_arm_crc32ch: 4686 CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break; 4687 case AArch64::BI__builtin_arm_crc32w: 4688 CRCIntrinsicID = Intrinsic::aarch64_crc32w; break; 4689 case AArch64::BI__builtin_arm_crc32cw: 4690 CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break; 4691 case AArch64::BI__builtin_arm_crc32d: 4692 CRCIntrinsicID = Intrinsic::aarch64_crc32x; break; 4693 case AArch64::BI__builtin_arm_crc32cd: 4694 CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break; 4695 } 4696 4697 if (CRCIntrinsicID != Intrinsic::not_intrinsic) { 4698 Value *Arg0 = EmitScalarExpr(E->getArg(0)); 4699 Value *Arg1 = EmitScalarExpr(E->getArg(1)); 4700 Function *F = CGM.getIntrinsic(CRCIntrinsicID); 4701 4702 llvm::Type *DataTy = F->getFunctionType()->getParamType(1); 4703 Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy); 4704 4705 return Builder.CreateCall(F, {Arg0, Arg1}); 4706 } 4707 4708 if (BuiltinID == AArch64::BI__builtin_arm_rsr || 4709 BuiltinID == AArch64::BI__builtin_arm_rsr64 || 4710 BuiltinID == AArch64::BI__builtin_arm_rsrp || 4711 BuiltinID == AArch64::BI__builtin_arm_wsr || 4712 BuiltinID == AArch64::BI__builtin_arm_wsr64 || 4713 BuiltinID == AArch64::BI__builtin_arm_wsrp) { 4714 4715 bool IsRead = BuiltinID == AArch64::BI__builtin_arm_rsr || 4716 BuiltinID == AArch64::BI__builtin_arm_rsr64 || 4717 BuiltinID == AArch64::BI__builtin_arm_rsrp; 4718 4719 bool IsPointerBuiltin = BuiltinID == AArch64::BI__builtin_arm_rsrp || 4720 BuiltinID == AArch64::BI__builtin_arm_wsrp; 4721 4722 bool Is64Bit = BuiltinID != AArch64::BI__builtin_arm_rsr && 4723 BuiltinID != AArch64::BI__builtin_arm_wsr; 4724 4725 llvm::Type *ValueType; 4726 llvm::Type *RegisterType = Int64Ty; 4727 if (IsPointerBuiltin) { 4728 ValueType = VoidPtrTy; 4729 } else if (Is64Bit) { 4730 ValueType = Int64Ty; 4731 } else { 4732 ValueType = Int32Ty; 4733 } 4734 4735 return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType, IsRead); 4736 } 4737 4738 // Find out if any arguments are required to be integer constant 4739 // expressions. 4740 unsigned ICEArguments = 0; 4741 ASTContext::GetBuiltinTypeError Error; 4742 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 4743 assert(Error == ASTContext::GE_None && "Should not codegen an error"); 4744 4745 llvm::SmallVector<Value*, 4> Ops; 4746 for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) { 4747 if ((ICEArguments & (1 << i)) == 0) { 4748 Ops.push_back(EmitScalarExpr(E->getArg(i))); 4749 } else { 4750 // If this is required to be a constant, constant fold it so that we know 4751 // that the generated intrinsic gets a ConstantInt. 4752 llvm::APSInt Result; 4753 bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext()); 4754 assert(IsConst && "Constant arg isn't actually constant?"); 4755 (void)IsConst; 4756 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); 4757 } 4758 } 4759 4760 auto SISDMap = makeArrayRef(AArch64SISDIntrinsicMap); 4761 const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap( 4762 SISDMap, BuiltinID, AArch64SISDIntrinsicsProvenSorted); 4763 4764 if (Builtin) { 4765 Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1))); 4766 Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E); 4767 assert(Result && "SISD intrinsic should have been handled"); 4768 return Result; 4769 } 4770 4771 llvm::APSInt Result; 4772 const Expr *Arg = E->getArg(E->getNumArgs()-1); 4773 NeonTypeFlags Type(0); 4774 if (Arg->isIntegerConstantExpr(Result, getContext())) 4775 // Determine the type of this overloaded NEON intrinsic. 4776 Type = NeonTypeFlags(Result.getZExtValue()); 4777 4778 bool usgn = Type.isUnsigned(); 4779 bool quad = Type.isQuad(); 4780 4781 // Handle non-overloaded intrinsics first. 4782 switch (BuiltinID) { 4783 default: break; 4784 case NEON::BI__builtin_neon_vldrq_p128: { 4785 llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128); 4786 Value *Ptr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int128PTy); 4787 return Builder.CreateDefaultAlignedLoad(Ptr); 4788 } 4789 case NEON::BI__builtin_neon_vstrq_p128: { 4790 llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128); 4791 Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy); 4792 return Builder.CreateDefaultAlignedStore(EmitScalarExpr(E->getArg(1)), Ptr); 4793 } 4794 case NEON::BI__builtin_neon_vcvts_u32_f32: 4795 case NEON::BI__builtin_neon_vcvtd_u64_f64: 4796 usgn = true; 4797 // FALL THROUGH 4798 case NEON::BI__builtin_neon_vcvts_s32_f32: 4799 case NEON::BI__builtin_neon_vcvtd_s64_f64: { 4800 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4801 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; 4802 llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty; 4803 llvm::Type *FTy = Is64 ? DoubleTy : FloatTy; 4804 Ops[0] = Builder.CreateBitCast(Ops[0], FTy); 4805 if (usgn) 4806 return Builder.CreateFPToUI(Ops[0], InTy); 4807 return Builder.CreateFPToSI(Ops[0], InTy); 4808 } 4809 case NEON::BI__builtin_neon_vcvts_f32_u32: 4810 case NEON::BI__builtin_neon_vcvtd_f64_u64: 4811 usgn = true; 4812 // FALL THROUGH 4813 case NEON::BI__builtin_neon_vcvts_f32_s32: 4814 case NEON::BI__builtin_neon_vcvtd_f64_s64: { 4815 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4816 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; 4817 llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty; 4818 llvm::Type *FTy = Is64 ? DoubleTy : FloatTy; 4819 Ops[0] = Builder.CreateBitCast(Ops[0], InTy); 4820 if (usgn) 4821 return Builder.CreateUIToFP(Ops[0], FTy); 4822 return Builder.CreateSIToFP(Ops[0], FTy); 4823 } 4824 case NEON::BI__builtin_neon_vpaddd_s64: { 4825 llvm::Type *Ty = llvm::VectorType::get(Int64Ty, 2); 4826 Value *Vec = EmitScalarExpr(E->getArg(0)); 4827 // The vector is v2f64, so make sure it's bitcast to that. 4828 Vec = Builder.CreateBitCast(Vec, Ty, "v2i64"); 4829 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); 4830 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); 4831 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 4832 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 4833 // Pairwise addition of a v2f64 into a scalar f64. 4834 return Builder.CreateAdd(Op0, Op1, "vpaddd"); 4835 } 4836 case NEON::BI__builtin_neon_vpaddd_f64: { 4837 llvm::Type *Ty = 4838 llvm::VectorType::get(DoubleTy, 2); 4839 Value *Vec = EmitScalarExpr(E->getArg(0)); 4840 // The vector is v2f64, so make sure it's bitcast to that. 4841 Vec = Builder.CreateBitCast(Vec, Ty, "v2f64"); 4842 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); 4843 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); 4844 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 4845 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 4846 // Pairwise addition of a v2f64 into a scalar f64. 4847 return Builder.CreateFAdd(Op0, Op1, "vpaddd"); 4848 } 4849 case NEON::BI__builtin_neon_vpadds_f32: { 4850 llvm::Type *Ty = 4851 llvm::VectorType::get(FloatTy, 2); 4852 Value *Vec = EmitScalarExpr(E->getArg(0)); 4853 // The vector is v2f32, so make sure it's bitcast to that. 4854 Vec = Builder.CreateBitCast(Vec, Ty, "v2f32"); 4855 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); 4856 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); 4857 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 4858 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 4859 // Pairwise addition of a v2f32 into a scalar f32. 4860 return Builder.CreateFAdd(Op0, Op1, "vpaddd"); 4861 } 4862 case NEON::BI__builtin_neon_vceqzd_s64: 4863 case NEON::BI__builtin_neon_vceqzd_f64: 4864 case NEON::BI__builtin_neon_vceqzs_f32: 4865 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4866 return EmitAArch64CompareBuiltinExpr( 4867 Ops[0], ConvertType(E->getCallReturnType(getContext())), 4868 ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ, "vceqz"); 4869 case NEON::BI__builtin_neon_vcgezd_s64: 4870 case NEON::BI__builtin_neon_vcgezd_f64: 4871 case NEON::BI__builtin_neon_vcgezs_f32: 4872 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4873 return EmitAArch64CompareBuiltinExpr( 4874 Ops[0], ConvertType(E->getCallReturnType(getContext())), 4875 ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE, "vcgez"); 4876 case NEON::BI__builtin_neon_vclezd_s64: 4877 case NEON::BI__builtin_neon_vclezd_f64: 4878 case NEON::BI__builtin_neon_vclezs_f32: 4879 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4880 return EmitAArch64CompareBuiltinExpr( 4881 Ops[0], ConvertType(E->getCallReturnType(getContext())), 4882 ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE, "vclez"); 4883 case NEON::BI__builtin_neon_vcgtzd_s64: 4884 case NEON::BI__builtin_neon_vcgtzd_f64: 4885 case NEON::BI__builtin_neon_vcgtzs_f32: 4886 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4887 return EmitAArch64CompareBuiltinExpr( 4888 Ops[0], ConvertType(E->getCallReturnType(getContext())), 4889 ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT, "vcgtz"); 4890 case NEON::BI__builtin_neon_vcltzd_s64: 4891 case NEON::BI__builtin_neon_vcltzd_f64: 4892 case NEON::BI__builtin_neon_vcltzs_f32: 4893 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4894 return EmitAArch64CompareBuiltinExpr( 4895 Ops[0], ConvertType(E->getCallReturnType(getContext())), 4896 ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT, "vcltz"); 4897 4898 case NEON::BI__builtin_neon_vceqzd_u64: { 4899 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4900 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); 4901 Ops[0] = 4902 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty)); 4903 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqzd"); 4904 } 4905 case NEON::BI__builtin_neon_vceqd_f64: 4906 case NEON::BI__builtin_neon_vcled_f64: 4907 case NEON::BI__builtin_neon_vcltd_f64: 4908 case NEON::BI__builtin_neon_vcged_f64: 4909 case NEON::BI__builtin_neon_vcgtd_f64: { 4910 llvm::CmpInst::Predicate P; 4911 switch (BuiltinID) { 4912 default: llvm_unreachable("missing builtin ID in switch!"); 4913 case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ; break; 4914 case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE; break; 4915 case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT; break; 4916 case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE; break; 4917 case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT; break; 4918 } 4919 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4920 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 4921 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); 4922 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); 4923 return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd"); 4924 } 4925 case NEON::BI__builtin_neon_vceqs_f32: 4926 case NEON::BI__builtin_neon_vcles_f32: 4927 case NEON::BI__builtin_neon_vclts_f32: 4928 case NEON::BI__builtin_neon_vcges_f32: 4929 case NEON::BI__builtin_neon_vcgts_f32: { 4930 llvm::CmpInst::Predicate P; 4931 switch (BuiltinID) { 4932 default: llvm_unreachable("missing builtin ID in switch!"); 4933 case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ; break; 4934 case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE; break; 4935 case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT; break; 4936 case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE; break; 4937 case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT; break; 4938 } 4939 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4940 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy); 4941 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy); 4942 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); 4943 return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd"); 4944 } 4945 case NEON::BI__builtin_neon_vceqd_s64: 4946 case NEON::BI__builtin_neon_vceqd_u64: 4947 case NEON::BI__builtin_neon_vcgtd_s64: 4948 case NEON::BI__builtin_neon_vcgtd_u64: 4949 case NEON::BI__builtin_neon_vcltd_s64: 4950 case NEON::BI__builtin_neon_vcltd_u64: 4951 case NEON::BI__builtin_neon_vcged_u64: 4952 case NEON::BI__builtin_neon_vcged_s64: 4953 case NEON::BI__builtin_neon_vcled_u64: 4954 case NEON::BI__builtin_neon_vcled_s64: { 4955 llvm::CmpInst::Predicate P; 4956 switch (BuiltinID) { 4957 default: llvm_unreachable("missing builtin ID in switch!"); 4958 case NEON::BI__builtin_neon_vceqd_s64: 4959 case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;break; 4960 case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;break; 4961 case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;break; 4962 case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;break; 4963 case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;break; 4964 case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;break; 4965 case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;break; 4966 case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;break; 4967 case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;break; 4968 } 4969 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4970 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); 4971 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); 4972 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]); 4973 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd"); 4974 } 4975 case NEON::BI__builtin_neon_vtstd_s64: 4976 case NEON::BI__builtin_neon_vtstd_u64: { 4977 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4978 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); 4979 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); 4980 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); 4981 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], 4982 llvm::Constant::getNullValue(Int64Ty)); 4983 return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd"); 4984 } 4985 case NEON::BI__builtin_neon_vset_lane_i8: 4986 case NEON::BI__builtin_neon_vset_lane_i16: 4987 case NEON::BI__builtin_neon_vset_lane_i32: 4988 case NEON::BI__builtin_neon_vset_lane_i64: 4989 case NEON::BI__builtin_neon_vset_lane_f32: 4990 case NEON::BI__builtin_neon_vsetq_lane_i8: 4991 case NEON::BI__builtin_neon_vsetq_lane_i16: 4992 case NEON::BI__builtin_neon_vsetq_lane_i32: 4993 case NEON::BI__builtin_neon_vsetq_lane_i64: 4994 case NEON::BI__builtin_neon_vsetq_lane_f32: 4995 Ops.push_back(EmitScalarExpr(E->getArg(2))); 4996 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 4997 case NEON::BI__builtin_neon_vset_lane_f64: 4998 // The vector type needs a cast for the v1f64 variant. 4999 Ops[1] = Builder.CreateBitCast(Ops[1], 5000 llvm::VectorType::get(DoubleTy, 1)); 5001 Ops.push_back(EmitScalarExpr(E->getArg(2))); 5002 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 5003 case NEON::BI__builtin_neon_vsetq_lane_f64: 5004 // The vector type needs a cast for the v2f64 variant. 5005 Ops[1] = Builder.CreateBitCast(Ops[1], 5006 llvm::VectorType::get(DoubleTy, 2)); 5007 Ops.push_back(EmitScalarExpr(E->getArg(2))); 5008 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 5009 5010 case NEON::BI__builtin_neon_vget_lane_i8: 5011 case NEON::BI__builtin_neon_vdupb_lane_i8: 5012 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 8)); 5013 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5014 "vget_lane"); 5015 case NEON::BI__builtin_neon_vgetq_lane_i8: 5016 case NEON::BI__builtin_neon_vdupb_laneq_i8: 5017 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 16)); 5018 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5019 "vgetq_lane"); 5020 case NEON::BI__builtin_neon_vget_lane_i16: 5021 case NEON::BI__builtin_neon_vduph_lane_i16: 5022 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 4)); 5023 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5024 "vget_lane"); 5025 case NEON::BI__builtin_neon_vgetq_lane_i16: 5026 case NEON::BI__builtin_neon_vduph_laneq_i16: 5027 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 8)); 5028 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5029 "vgetq_lane"); 5030 case NEON::BI__builtin_neon_vget_lane_i32: 5031 case NEON::BI__builtin_neon_vdups_lane_i32: 5032 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 2)); 5033 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5034 "vget_lane"); 5035 case NEON::BI__builtin_neon_vdups_lane_f32: 5036 Ops[0] = Builder.CreateBitCast(Ops[0], 5037 llvm::VectorType::get(FloatTy, 2)); 5038 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5039 "vdups_lane"); 5040 case NEON::BI__builtin_neon_vgetq_lane_i32: 5041 case NEON::BI__builtin_neon_vdups_laneq_i32: 5042 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4)); 5043 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5044 "vgetq_lane"); 5045 case NEON::BI__builtin_neon_vget_lane_i64: 5046 case NEON::BI__builtin_neon_vdupd_lane_i64: 5047 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 1)); 5048 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5049 "vget_lane"); 5050 case NEON::BI__builtin_neon_vdupd_lane_f64: 5051 Ops[0] = Builder.CreateBitCast(Ops[0], 5052 llvm::VectorType::get(DoubleTy, 1)); 5053 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5054 "vdupd_lane"); 5055 case NEON::BI__builtin_neon_vgetq_lane_i64: 5056 case NEON::BI__builtin_neon_vdupd_laneq_i64: 5057 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2)); 5058 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5059 "vgetq_lane"); 5060 case NEON::BI__builtin_neon_vget_lane_f32: 5061 Ops[0] = Builder.CreateBitCast(Ops[0], 5062 llvm::VectorType::get(FloatTy, 2)); 5063 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5064 "vget_lane"); 5065 case NEON::BI__builtin_neon_vget_lane_f64: 5066 Ops[0] = Builder.CreateBitCast(Ops[0], 5067 llvm::VectorType::get(DoubleTy, 1)); 5068 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5069 "vget_lane"); 5070 case NEON::BI__builtin_neon_vgetq_lane_f32: 5071 case NEON::BI__builtin_neon_vdups_laneq_f32: 5072 Ops[0] = Builder.CreateBitCast(Ops[0], 5073 llvm::VectorType::get(FloatTy, 4)); 5074 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5075 "vgetq_lane"); 5076 case NEON::BI__builtin_neon_vgetq_lane_f64: 5077 case NEON::BI__builtin_neon_vdupd_laneq_f64: 5078 Ops[0] = Builder.CreateBitCast(Ops[0], 5079 llvm::VectorType::get(DoubleTy, 2)); 5080 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5081 "vgetq_lane"); 5082 case NEON::BI__builtin_neon_vaddd_s64: 5083 case NEON::BI__builtin_neon_vaddd_u64: 5084 return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd"); 5085 case NEON::BI__builtin_neon_vsubd_s64: 5086 case NEON::BI__builtin_neon_vsubd_u64: 5087 return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd"); 5088 case NEON::BI__builtin_neon_vqdmlalh_s16: 5089 case NEON::BI__builtin_neon_vqdmlslh_s16: { 5090 SmallVector<Value *, 2> ProductOps; 5091 ProductOps.push_back(vectorWrapScalar16(Ops[1])); 5092 ProductOps.push_back(vectorWrapScalar16(EmitScalarExpr(E->getArg(2)))); 5093 llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4); 5094 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), 5095 ProductOps, "vqdmlXl"); 5096 Constant *CI = ConstantInt::get(SizeTy, 0); 5097 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); 5098 5099 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16 5100 ? Intrinsic::aarch64_neon_sqadd 5101 : Intrinsic::aarch64_neon_sqsub; 5102 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl"); 5103 } 5104 case NEON::BI__builtin_neon_vqshlud_n_s64: { 5105 Ops.push_back(EmitScalarExpr(E->getArg(1))); 5106 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); 5107 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqshlu, Int64Ty), 5108 Ops, "vqshlu_n"); 5109 } 5110 case NEON::BI__builtin_neon_vqshld_n_u64: 5111 case NEON::BI__builtin_neon_vqshld_n_s64: { 5112 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64 5113 ? Intrinsic::aarch64_neon_uqshl 5114 : Intrinsic::aarch64_neon_sqshl; 5115 Ops.push_back(EmitScalarExpr(E->getArg(1))); 5116 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); 5117 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n"); 5118 } 5119 case NEON::BI__builtin_neon_vrshrd_n_u64: 5120 case NEON::BI__builtin_neon_vrshrd_n_s64: { 5121 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64 5122 ? Intrinsic::aarch64_neon_urshl 5123 : Intrinsic::aarch64_neon_srshl; 5124 Ops.push_back(EmitScalarExpr(E->getArg(1))); 5125 int SV = cast<ConstantInt>(Ops[1])->getSExtValue(); 5126 Ops[1] = ConstantInt::get(Int64Ty, -SV); 5127 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n"); 5128 } 5129 case NEON::BI__builtin_neon_vrsrad_n_u64: 5130 case NEON::BI__builtin_neon_vrsrad_n_s64: { 5131 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64 5132 ? Intrinsic::aarch64_neon_urshl 5133 : Intrinsic::aarch64_neon_srshl; 5134 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); 5135 Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2)))); 5136 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty), 5137 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)}); 5138 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty)); 5139 } 5140 case NEON::BI__builtin_neon_vshld_n_s64: 5141 case NEON::BI__builtin_neon_vshld_n_u64: { 5142 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 5143 return Builder.CreateShl( 5144 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n"); 5145 } 5146 case NEON::BI__builtin_neon_vshrd_n_s64: { 5147 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 5148 return Builder.CreateAShr( 5149 Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), 5150 Amt->getZExtValue())), 5151 "shrd_n"); 5152 } 5153 case NEON::BI__builtin_neon_vshrd_n_u64: { 5154 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 5155 uint64_t ShiftAmt = Amt->getZExtValue(); 5156 // Right-shifting an unsigned value by its size yields 0. 5157 if (ShiftAmt == 64) 5158 return ConstantInt::get(Int64Ty, 0); 5159 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt), 5160 "shrd_n"); 5161 } 5162 case NEON::BI__builtin_neon_vsrad_n_s64: { 5163 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2))); 5164 Ops[1] = Builder.CreateAShr( 5165 Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), 5166 Amt->getZExtValue())), 5167 "shrd_n"); 5168 return Builder.CreateAdd(Ops[0], Ops[1]); 5169 } 5170 case NEON::BI__builtin_neon_vsrad_n_u64: { 5171 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2))); 5172 uint64_t ShiftAmt = Amt->getZExtValue(); 5173 // Right-shifting an unsigned value by its size yields 0. 5174 // As Op + 0 = Op, return Ops[0] directly. 5175 if (ShiftAmt == 64) 5176 return Ops[0]; 5177 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt), 5178 "shrd_n"); 5179 return Builder.CreateAdd(Ops[0], Ops[1]); 5180 } 5181 case NEON::BI__builtin_neon_vqdmlalh_lane_s16: 5182 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16: 5183 case NEON::BI__builtin_neon_vqdmlslh_lane_s16: 5184 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: { 5185 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), 5186 "lane"); 5187 SmallVector<Value *, 2> ProductOps; 5188 ProductOps.push_back(vectorWrapScalar16(Ops[1])); 5189 ProductOps.push_back(vectorWrapScalar16(Ops[2])); 5190 llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4); 5191 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), 5192 ProductOps, "vqdmlXl"); 5193 Constant *CI = ConstantInt::get(SizeTy, 0); 5194 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); 5195 Ops.pop_back(); 5196 5197 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 || 5198 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16) 5199 ? Intrinsic::aarch64_neon_sqadd 5200 : Intrinsic::aarch64_neon_sqsub; 5201 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl"); 5202 } 5203 case NEON::BI__builtin_neon_vqdmlals_s32: 5204 case NEON::BI__builtin_neon_vqdmlsls_s32: { 5205 SmallVector<Value *, 2> ProductOps; 5206 ProductOps.push_back(Ops[1]); 5207 ProductOps.push_back(EmitScalarExpr(E->getArg(2))); 5208 Ops[1] = 5209 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar), 5210 ProductOps, "vqdmlXl"); 5211 5212 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32 5213 ? Intrinsic::aarch64_neon_sqadd 5214 : Intrinsic::aarch64_neon_sqsub; 5215 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl"); 5216 } 5217 case NEON::BI__builtin_neon_vqdmlals_lane_s32: 5218 case NEON::BI__builtin_neon_vqdmlals_laneq_s32: 5219 case NEON::BI__builtin_neon_vqdmlsls_lane_s32: 5220 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: { 5221 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), 5222 "lane"); 5223 SmallVector<Value *, 2> ProductOps; 5224 ProductOps.push_back(Ops[1]); 5225 ProductOps.push_back(Ops[2]); 5226 Ops[1] = 5227 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar), 5228 ProductOps, "vqdmlXl"); 5229 Ops.pop_back(); 5230 5231 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 || 5232 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32) 5233 ? Intrinsic::aarch64_neon_sqadd 5234 : Intrinsic::aarch64_neon_sqsub; 5235 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl"); 5236 } 5237 } 5238 5239 llvm::VectorType *VTy = GetNeonType(this, Type); 5240 llvm::Type *Ty = VTy; 5241 if (!Ty) 5242 return nullptr; 5243 5244 // Not all intrinsics handled by the common case work for AArch64 yet, so only 5245 // defer to common code if it's been added to our special map. 5246 Builtin = findNeonIntrinsicInMap(AArch64SIMDIntrinsicMap, BuiltinID, 5247 AArch64SIMDIntrinsicsProvenSorted); 5248 5249 if (Builtin) 5250 return EmitCommonNeonBuiltinExpr( 5251 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic, 5252 Builtin->NameHint, Builtin->TypeModifier, E, Ops, 5253 /*never use addresses*/ Address::invalid(), Address::invalid()); 5254 5255 if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops)) 5256 return V; 5257 5258 unsigned Int; 5259 switch (BuiltinID) { 5260 default: return nullptr; 5261 case NEON::BI__builtin_neon_vbsl_v: 5262 case NEON::BI__builtin_neon_vbslq_v: { 5263 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy); 5264 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl"); 5265 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl"); 5266 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl"); 5267 5268 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl"); 5269 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl"); 5270 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl"); 5271 return Builder.CreateBitCast(Ops[0], Ty); 5272 } 5273 case NEON::BI__builtin_neon_vfma_lane_v: 5274 case NEON::BI__builtin_neon_vfmaq_lane_v: { // Only used for FP types 5275 // The ARM builtins (and instructions) have the addend as the first 5276 // operand, but the 'fma' intrinsics have it last. Swap it around here. 5277 Value *Addend = Ops[0]; 5278 Value *Multiplicand = Ops[1]; 5279 Value *LaneSource = Ops[2]; 5280 Ops[0] = Multiplicand; 5281 Ops[1] = LaneSource; 5282 Ops[2] = Addend; 5283 5284 // Now adjust things to handle the lane access. 5285 llvm::Type *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v ? 5286 llvm::VectorType::get(VTy->getElementType(), VTy->getNumElements() / 2) : 5287 VTy; 5288 llvm::Constant *cst = cast<Constant>(Ops[3]); 5289 Value *SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), cst); 5290 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy); 5291 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane"); 5292 5293 Ops.pop_back(); 5294 Int = Intrinsic::fma; 5295 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla"); 5296 } 5297 case NEON::BI__builtin_neon_vfma_laneq_v: { 5298 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); 5299 // v1f64 fma should be mapped to Neon scalar f64 fma 5300 if (VTy && VTy->getElementType() == DoubleTy) { 5301 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 5302 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); 5303 llvm::Type *VTy = GetNeonType(this, 5304 NeonTypeFlags(NeonTypeFlags::Float64, false, true)); 5305 Ops[2] = Builder.CreateBitCast(Ops[2], VTy); 5306 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); 5307 Value *F = CGM.getIntrinsic(Intrinsic::fma, DoubleTy); 5308 Value *Result = Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); 5309 return Builder.CreateBitCast(Result, Ty); 5310 } 5311 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 5312 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5313 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5314 5315 llvm::Type *STy = llvm::VectorType::get(VTy->getElementType(), 5316 VTy->getNumElements() * 2); 5317 Ops[2] = Builder.CreateBitCast(Ops[2], STy); 5318 Value* SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), 5319 cast<ConstantInt>(Ops[3])); 5320 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane"); 5321 5322 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]}); 5323 } 5324 case NEON::BI__builtin_neon_vfmaq_laneq_v: { 5325 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 5326 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5327 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5328 5329 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 5330 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3])); 5331 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]}); 5332 } 5333 case NEON::BI__builtin_neon_vfmas_lane_f32: 5334 case NEON::BI__builtin_neon_vfmas_laneq_f32: 5335 case NEON::BI__builtin_neon_vfmad_lane_f64: 5336 case NEON::BI__builtin_neon_vfmad_laneq_f64: { 5337 Ops.push_back(EmitScalarExpr(E->getArg(3))); 5338 llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext())); 5339 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 5340 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); 5341 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); 5342 } 5343 case NEON::BI__builtin_neon_vmull_v: 5344 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 5345 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull; 5346 if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull; 5347 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); 5348 case NEON::BI__builtin_neon_vmax_v: 5349 case NEON::BI__builtin_neon_vmaxq_v: 5350 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 5351 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax; 5352 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax; 5353 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax"); 5354 case NEON::BI__builtin_neon_vmin_v: 5355 case NEON::BI__builtin_neon_vminq_v: 5356 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 5357 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin; 5358 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin; 5359 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin"); 5360 case NEON::BI__builtin_neon_vabd_v: 5361 case NEON::BI__builtin_neon_vabdq_v: 5362 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 5363 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd; 5364 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd; 5365 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd"); 5366 case NEON::BI__builtin_neon_vpadal_v: 5367 case NEON::BI__builtin_neon_vpadalq_v: { 5368 unsigned ArgElts = VTy->getNumElements(); 5369 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType()); 5370 unsigned BitWidth = EltTy->getBitWidth(); 5371 llvm::Type *ArgTy = llvm::VectorType::get( 5372 llvm::IntegerType::get(getLLVMContext(), BitWidth/2), 2*ArgElts); 5373 llvm::Type* Tys[2] = { VTy, ArgTy }; 5374 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp; 5375 SmallVector<llvm::Value*, 1> TmpOps; 5376 TmpOps.push_back(Ops[1]); 5377 Function *F = CGM.getIntrinsic(Int, Tys); 5378 llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vpadal"); 5379 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType()); 5380 return Builder.CreateAdd(tmp, addend); 5381 } 5382 case NEON::BI__builtin_neon_vpmin_v: 5383 case NEON::BI__builtin_neon_vpminq_v: 5384 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 5385 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp; 5386 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp; 5387 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin"); 5388 case NEON::BI__builtin_neon_vpmax_v: 5389 case NEON::BI__builtin_neon_vpmaxq_v: 5390 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 5391 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp; 5392 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp; 5393 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax"); 5394 case NEON::BI__builtin_neon_vminnm_v: 5395 case NEON::BI__builtin_neon_vminnmq_v: 5396 Int = Intrinsic::aarch64_neon_fminnm; 5397 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm"); 5398 case NEON::BI__builtin_neon_vmaxnm_v: 5399 case NEON::BI__builtin_neon_vmaxnmq_v: 5400 Int = Intrinsic::aarch64_neon_fmaxnm; 5401 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm"); 5402 case NEON::BI__builtin_neon_vrecpss_f32: { 5403 Ops.push_back(EmitScalarExpr(E->getArg(1))); 5404 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, FloatTy), 5405 Ops, "vrecps"); 5406 } 5407 case NEON::BI__builtin_neon_vrecpsd_f64: { 5408 Ops.push_back(EmitScalarExpr(E->getArg(1))); 5409 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, DoubleTy), 5410 Ops, "vrecps"); 5411 } 5412 case NEON::BI__builtin_neon_vqshrun_n_v: 5413 Int = Intrinsic::aarch64_neon_sqshrun; 5414 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n"); 5415 case NEON::BI__builtin_neon_vqrshrun_n_v: 5416 Int = Intrinsic::aarch64_neon_sqrshrun; 5417 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n"); 5418 case NEON::BI__builtin_neon_vqshrn_n_v: 5419 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn; 5420 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n"); 5421 case NEON::BI__builtin_neon_vrshrn_n_v: 5422 Int = Intrinsic::aarch64_neon_rshrn; 5423 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n"); 5424 case NEON::BI__builtin_neon_vqrshrn_n_v: 5425 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn; 5426 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n"); 5427 case NEON::BI__builtin_neon_vrnda_v: 5428 case NEON::BI__builtin_neon_vrndaq_v: { 5429 Int = Intrinsic::round; 5430 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda"); 5431 } 5432 case NEON::BI__builtin_neon_vrndi_v: 5433 case NEON::BI__builtin_neon_vrndiq_v: { 5434 Int = Intrinsic::nearbyint; 5435 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndi"); 5436 } 5437 case NEON::BI__builtin_neon_vrndm_v: 5438 case NEON::BI__builtin_neon_vrndmq_v: { 5439 Int = Intrinsic::floor; 5440 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm"); 5441 } 5442 case NEON::BI__builtin_neon_vrndn_v: 5443 case NEON::BI__builtin_neon_vrndnq_v: { 5444 Int = Intrinsic::aarch64_neon_frintn; 5445 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn"); 5446 } 5447 case NEON::BI__builtin_neon_vrndp_v: 5448 case NEON::BI__builtin_neon_vrndpq_v: { 5449 Int = Intrinsic::ceil; 5450 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp"); 5451 } 5452 case NEON::BI__builtin_neon_vrndx_v: 5453 case NEON::BI__builtin_neon_vrndxq_v: { 5454 Int = Intrinsic::rint; 5455 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx"); 5456 } 5457 case NEON::BI__builtin_neon_vrnd_v: 5458 case NEON::BI__builtin_neon_vrndq_v: { 5459 Int = Intrinsic::trunc; 5460 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz"); 5461 } 5462 case NEON::BI__builtin_neon_vceqz_v: 5463 case NEON::BI__builtin_neon_vceqzq_v: 5464 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ, 5465 ICmpInst::ICMP_EQ, "vceqz"); 5466 case NEON::BI__builtin_neon_vcgez_v: 5467 case NEON::BI__builtin_neon_vcgezq_v: 5468 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE, 5469 ICmpInst::ICMP_SGE, "vcgez"); 5470 case NEON::BI__builtin_neon_vclez_v: 5471 case NEON::BI__builtin_neon_vclezq_v: 5472 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE, 5473 ICmpInst::ICMP_SLE, "vclez"); 5474 case NEON::BI__builtin_neon_vcgtz_v: 5475 case NEON::BI__builtin_neon_vcgtzq_v: 5476 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT, 5477 ICmpInst::ICMP_SGT, "vcgtz"); 5478 case NEON::BI__builtin_neon_vcltz_v: 5479 case NEON::BI__builtin_neon_vcltzq_v: 5480 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT, 5481 ICmpInst::ICMP_SLT, "vcltz"); 5482 case NEON::BI__builtin_neon_vcvt_f64_v: 5483 case NEON::BI__builtin_neon_vcvtq_f64_v: 5484 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5485 Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad)); 5486 return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") 5487 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); 5488 case NEON::BI__builtin_neon_vcvt_f64_f32: { 5489 assert(Type.getEltType() == NeonTypeFlags::Float64 && quad && 5490 "unexpected vcvt_f64_f32 builtin"); 5491 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float32, false, false); 5492 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); 5493 5494 return Builder.CreateFPExt(Ops[0], Ty, "vcvt"); 5495 } 5496 case NEON::BI__builtin_neon_vcvt_f32_f64: { 5497 assert(Type.getEltType() == NeonTypeFlags::Float32 && 5498 "unexpected vcvt_f32_f64 builtin"); 5499 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float64, false, true); 5500 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); 5501 5502 return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt"); 5503 } 5504 case NEON::BI__builtin_neon_vcvt_s32_v: 5505 case NEON::BI__builtin_neon_vcvt_u32_v: 5506 case NEON::BI__builtin_neon_vcvt_s64_v: 5507 case NEON::BI__builtin_neon_vcvt_u64_v: 5508 case NEON::BI__builtin_neon_vcvtq_s32_v: 5509 case NEON::BI__builtin_neon_vcvtq_u32_v: 5510 case NEON::BI__builtin_neon_vcvtq_s64_v: 5511 case NEON::BI__builtin_neon_vcvtq_u64_v: { 5512 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); 5513 if (usgn) 5514 return Builder.CreateFPToUI(Ops[0], Ty); 5515 return Builder.CreateFPToSI(Ops[0], Ty); 5516 } 5517 case NEON::BI__builtin_neon_vcvta_s32_v: 5518 case NEON::BI__builtin_neon_vcvtaq_s32_v: 5519 case NEON::BI__builtin_neon_vcvta_u32_v: 5520 case NEON::BI__builtin_neon_vcvtaq_u32_v: 5521 case NEON::BI__builtin_neon_vcvta_s64_v: 5522 case NEON::BI__builtin_neon_vcvtaq_s64_v: 5523 case NEON::BI__builtin_neon_vcvta_u64_v: 5524 case NEON::BI__builtin_neon_vcvtaq_u64_v: { 5525 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas; 5526 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 5527 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta"); 5528 } 5529 case NEON::BI__builtin_neon_vcvtm_s32_v: 5530 case NEON::BI__builtin_neon_vcvtmq_s32_v: 5531 case NEON::BI__builtin_neon_vcvtm_u32_v: 5532 case NEON::BI__builtin_neon_vcvtmq_u32_v: 5533 case NEON::BI__builtin_neon_vcvtm_s64_v: 5534 case NEON::BI__builtin_neon_vcvtmq_s64_v: 5535 case NEON::BI__builtin_neon_vcvtm_u64_v: 5536 case NEON::BI__builtin_neon_vcvtmq_u64_v: { 5537 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms; 5538 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 5539 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm"); 5540 } 5541 case NEON::BI__builtin_neon_vcvtn_s32_v: 5542 case NEON::BI__builtin_neon_vcvtnq_s32_v: 5543 case NEON::BI__builtin_neon_vcvtn_u32_v: 5544 case NEON::BI__builtin_neon_vcvtnq_u32_v: 5545 case NEON::BI__builtin_neon_vcvtn_s64_v: 5546 case NEON::BI__builtin_neon_vcvtnq_s64_v: 5547 case NEON::BI__builtin_neon_vcvtn_u64_v: 5548 case NEON::BI__builtin_neon_vcvtnq_u64_v: { 5549 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns; 5550 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 5551 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn"); 5552 } 5553 case NEON::BI__builtin_neon_vcvtp_s32_v: 5554 case NEON::BI__builtin_neon_vcvtpq_s32_v: 5555 case NEON::BI__builtin_neon_vcvtp_u32_v: 5556 case NEON::BI__builtin_neon_vcvtpq_u32_v: 5557 case NEON::BI__builtin_neon_vcvtp_s64_v: 5558 case NEON::BI__builtin_neon_vcvtpq_s64_v: 5559 case NEON::BI__builtin_neon_vcvtp_u64_v: 5560 case NEON::BI__builtin_neon_vcvtpq_u64_v: { 5561 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps; 5562 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 5563 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp"); 5564 } 5565 case NEON::BI__builtin_neon_vmulx_v: 5566 case NEON::BI__builtin_neon_vmulxq_v: { 5567 Int = Intrinsic::aarch64_neon_fmulx; 5568 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx"); 5569 } 5570 case NEON::BI__builtin_neon_vmul_lane_v: 5571 case NEON::BI__builtin_neon_vmul_laneq_v: { 5572 // v1f64 vmul_lane should be mapped to Neon scalar mul lane 5573 bool Quad = false; 5574 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v) 5575 Quad = true; 5576 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 5577 llvm::Type *VTy = GetNeonType(this, 5578 NeonTypeFlags(NeonTypeFlags::Float64, false, Quad)); 5579 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); 5580 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract"); 5581 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]); 5582 return Builder.CreateBitCast(Result, Ty); 5583 } 5584 case NEON::BI__builtin_neon_vnegd_s64: 5585 return Builder.CreateNeg(EmitScalarExpr(E->getArg(0)), "vnegd"); 5586 case NEON::BI__builtin_neon_vpmaxnm_v: 5587 case NEON::BI__builtin_neon_vpmaxnmq_v: { 5588 Int = Intrinsic::aarch64_neon_fmaxnmp; 5589 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm"); 5590 } 5591 case NEON::BI__builtin_neon_vpminnm_v: 5592 case NEON::BI__builtin_neon_vpminnmq_v: { 5593 Int = Intrinsic::aarch64_neon_fminnmp; 5594 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm"); 5595 } 5596 case NEON::BI__builtin_neon_vsqrt_v: 5597 case NEON::BI__builtin_neon_vsqrtq_v: { 5598 Int = Intrinsic::sqrt; 5599 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5600 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt"); 5601 } 5602 case NEON::BI__builtin_neon_vrbit_v: 5603 case NEON::BI__builtin_neon_vrbitq_v: { 5604 Int = Intrinsic::aarch64_neon_rbit; 5605 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit"); 5606 } 5607 case NEON::BI__builtin_neon_vaddv_u8: 5608 // FIXME: These are handled by the AArch64 scalar code. 5609 usgn = true; 5610 // FALLTHROUGH 5611 case NEON::BI__builtin_neon_vaddv_s8: { 5612 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 5613 Ty = Int32Ty; 5614 VTy = llvm::VectorType::get(Int8Ty, 8); 5615 llvm::Type *Tys[2] = { Ty, VTy }; 5616 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5617 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 5618 return Builder.CreateTrunc(Ops[0], Int8Ty); 5619 } 5620 case NEON::BI__builtin_neon_vaddv_u16: 5621 usgn = true; 5622 // FALLTHROUGH 5623 case NEON::BI__builtin_neon_vaddv_s16: { 5624 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 5625 Ty = Int32Ty; 5626 VTy = llvm::VectorType::get(Int16Ty, 4); 5627 llvm::Type *Tys[2] = { Ty, VTy }; 5628 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5629 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 5630 return Builder.CreateTrunc(Ops[0], Int16Ty); 5631 } 5632 case NEON::BI__builtin_neon_vaddvq_u8: 5633 usgn = true; 5634 // FALLTHROUGH 5635 case NEON::BI__builtin_neon_vaddvq_s8: { 5636 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 5637 Ty = Int32Ty; 5638 VTy = llvm::VectorType::get(Int8Ty, 16); 5639 llvm::Type *Tys[2] = { Ty, VTy }; 5640 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5641 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 5642 return Builder.CreateTrunc(Ops[0], Int8Ty); 5643 } 5644 case NEON::BI__builtin_neon_vaddvq_u16: 5645 usgn = true; 5646 // FALLTHROUGH 5647 case NEON::BI__builtin_neon_vaddvq_s16: { 5648 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 5649 Ty = Int32Ty; 5650 VTy = llvm::VectorType::get(Int16Ty, 8); 5651 llvm::Type *Tys[2] = { Ty, VTy }; 5652 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5653 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 5654 return Builder.CreateTrunc(Ops[0], Int16Ty); 5655 } 5656 case NEON::BI__builtin_neon_vmaxv_u8: { 5657 Int = Intrinsic::aarch64_neon_umaxv; 5658 Ty = Int32Ty; 5659 VTy = llvm::VectorType::get(Int8Ty, 8); 5660 llvm::Type *Tys[2] = { Ty, VTy }; 5661 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5662 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5663 return Builder.CreateTrunc(Ops[0], Int8Ty); 5664 } 5665 case NEON::BI__builtin_neon_vmaxv_u16: { 5666 Int = Intrinsic::aarch64_neon_umaxv; 5667 Ty = Int32Ty; 5668 VTy = llvm::VectorType::get(Int16Ty, 4); 5669 llvm::Type *Tys[2] = { Ty, VTy }; 5670 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5671 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5672 return Builder.CreateTrunc(Ops[0], Int16Ty); 5673 } 5674 case NEON::BI__builtin_neon_vmaxvq_u8: { 5675 Int = Intrinsic::aarch64_neon_umaxv; 5676 Ty = Int32Ty; 5677 VTy = llvm::VectorType::get(Int8Ty, 16); 5678 llvm::Type *Tys[2] = { Ty, VTy }; 5679 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5680 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5681 return Builder.CreateTrunc(Ops[0], Int8Ty); 5682 } 5683 case NEON::BI__builtin_neon_vmaxvq_u16: { 5684 Int = Intrinsic::aarch64_neon_umaxv; 5685 Ty = Int32Ty; 5686 VTy = llvm::VectorType::get(Int16Ty, 8); 5687 llvm::Type *Tys[2] = { Ty, VTy }; 5688 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5689 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5690 return Builder.CreateTrunc(Ops[0], Int16Ty); 5691 } 5692 case NEON::BI__builtin_neon_vmaxv_s8: { 5693 Int = Intrinsic::aarch64_neon_smaxv; 5694 Ty = Int32Ty; 5695 VTy = llvm::VectorType::get(Int8Ty, 8); 5696 llvm::Type *Tys[2] = { Ty, VTy }; 5697 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5698 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5699 return Builder.CreateTrunc(Ops[0], Int8Ty); 5700 } 5701 case NEON::BI__builtin_neon_vmaxv_s16: { 5702 Int = Intrinsic::aarch64_neon_smaxv; 5703 Ty = Int32Ty; 5704 VTy = llvm::VectorType::get(Int16Ty, 4); 5705 llvm::Type *Tys[2] = { Ty, VTy }; 5706 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5707 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5708 return Builder.CreateTrunc(Ops[0], Int16Ty); 5709 } 5710 case NEON::BI__builtin_neon_vmaxvq_s8: { 5711 Int = Intrinsic::aarch64_neon_smaxv; 5712 Ty = Int32Ty; 5713 VTy = llvm::VectorType::get(Int8Ty, 16); 5714 llvm::Type *Tys[2] = { Ty, VTy }; 5715 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5716 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5717 return Builder.CreateTrunc(Ops[0], Int8Ty); 5718 } 5719 case NEON::BI__builtin_neon_vmaxvq_s16: { 5720 Int = Intrinsic::aarch64_neon_smaxv; 5721 Ty = Int32Ty; 5722 VTy = llvm::VectorType::get(Int16Ty, 8); 5723 llvm::Type *Tys[2] = { Ty, VTy }; 5724 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5725 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5726 return Builder.CreateTrunc(Ops[0], Int16Ty); 5727 } 5728 case NEON::BI__builtin_neon_vminv_u8: { 5729 Int = Intrinsic::aarch64_neon_uminv; 5730 Ty = Int32Ty; 5731 VTy = llvm::VectorType::get(Int8Ty, 8); 5732 llvm::Type *Tys[2] = { Ty, VTy }; 5733 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5734 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5735 return Builder.CreateTrunc(Ops[0], Int8Ty); 5736 } 5737 case NEON::BI__builtin_neon_vminv_u16: { 5738 Int = Intrinsic::aarch64_neon_uminv; 5739 Ty = Int32Ty; 5740 VTy = llvm::VectorType::get(Int16Ty, 4); 5741 llvm::Type *Tys[2] = { Ty, VTy }; 5742 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5743 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5744 return Builder.CreateTrunc(Ops[0], Int16Ty); 5745 } 5746 case NEON::BI__builtin_neon_vminvq_u8: { 5747 Int = Intrinsic::aarch64_neon_uminv; 5748 Ty = Int32Ty; 5749 VTy = llvm::VectorType::get(Int8Ty, 16); 5750 llvm::Type *Tys[2] = { Ty, VTy }; 5751 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5752 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5753 return Builder.CreateTrunc(Ops[0], Int8Ty); 5754 } 5755 case NEON::BI__builtin_neon_vminvq_u16: { 5756 Int = Intrinsic::aarch64_neon_uminv; 5757 Ty = Int32Ty; 5758 VTy = llvm::VectorType::get(Int16Ty, 8); 5759 llvm::Type *Tys[2] = { Ty, VTy }; 5760 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5761 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5762 return Builder.CreateTrunc(Ops[0], Int16Ty); 5763 } 5764 case NEON::BI__builtin_neon_vminv_s8: { 5765 Int = Intrinsic::aarch64_neon_sminv; 5766 Ty = Int32Ty; 5767 VTy = llvm::VectorType::get(Int8Ty, 8); 5768 llvm::Type *Tys[2] = { Ty, VTy }; 5769 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5770 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5771 return Builder.CreateTrunc(Ops[0], Int8Ty); 5772 } 5773 case NEON::BI__builtin_neon_vminv_s16: { 5774 Int = Intrinsic::aarch64_neon_sminv; 5775 Ty = Int32Ty; 5776 VTy = llvm::VectorType::get(Int16Ty, 4); 5777 llvm::Type *Tys[2] = { Ty, VTy }; 5778 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5779 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5780 return Builder.CreateTrunc(Ops[0], Int16Ty); 5781 } 5782 case NEON::BI__builtin_neon_vminvq_s8: { 5783 Int = Intrinsic::aarch64_neon_sminv; 5784 Ty = Int32Ty; 5785 VTy = llvm::VectorType::get(Int8Ty, 16); 5786 llvm::Type *Tys[2] = { Ty, VTy }; 5787 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5788 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5789 return Builder.CreateTrunc(Ops[0], Int8Ty); 5790 } 5791 case NEON::BI__builtin_neon_vminvq_s16: { 5792 Int = Intrinsic::aarch64_neon_sminv; 5793 Ty = Int32Ty; 5794 VTy = llvm::VectorType::get(Int16Ty, 8); 5795 llvm::Type *Tys[2] = { Ty, VTy }; 5796 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5797 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5798 return Builder.CreateTrunc(Ops[0], Int16Ty); 5799 } 5800 case NEON::BI__builtin_neon_vmul_n_f64: { 5801 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 5802 Value *RHS = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), DoubleTy); 5803 return Builder.CreateFMul(Ops[0], RHS); 5804 } 5805 case NEON::BI__builtin_neon_vaddlv_u8: { 5806 Int = Intrinsic::aarch64_neon_uaddlv; 5807 Ty = Int32Ty; 5808 VTy = llvm::VectorType::get(Int8Ty, 8); 5809 llvm::Type *Tys[2] = { Ty, VTy }; 5810 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5811 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5812 return Builder.CreateTrunc(Ops[0], Int16Ty); 5813 } 5814 case NEON::BI__builtin_neon_vaddlv_u16: { 5815 Int = Intrinsic::aarch64_neon_uaddlv; 5816 Ty = Int32Ty; 5817 VTy = llvm::VectorType::get(Int16Ty, 4); 5818 llvm::Type *Tys[2] = { Ty, VTy }; 5819 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5820 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5821 } 5822 case NEON::BI__builtin_neon_vaddlvq_u8: { 5823 Int = Intrinsic::aarch64_neon_uaddlv; 5824 Ty = Int32Ty; 5825 VTy = llvm::VectorType::get(Int8Ty, 16); 5826 llvm::Type *Tys[2] = { Ty, VTy }; 5827 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5828 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5829 return Builder.CreateTrunc(Ops[0], Int16Ty); 5830 } 5831 case NEON::BI__builtin_neon_vaddlvq_u16: { 5832 Int = Intrinsic::aarch64_neon_uaddlv; 5833 Ty = Int32Ty; 5834 VTy = llvm::VectorType::get(Int16Ty, 8); 5835 llvm::Type *Tys[2] = { Ty, VTy }; 5836 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5837 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5838 } 5839 case NEON::BI__builtin_neon_vaddlv_s8: { 5840 Int = Intrinsic::aarch64_neon_saddlv; 5841 Ty = Int32Ty; 5842 VTy = llvm::VectorType::get(Int8Ty, 8); 5843 llvm::Type *Tys[2] = { Ty, VTy }; 5844 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5845 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5846 return Builder.CreateTrunc(Ops[0], Int16Ty); 5847 } 5848 case NEON::BI__builtin_neon_vaddlv_s16: { 5849 Int = Intrinsic::aarch64_neon_saddlv; 5850 Ty = Int32Ty; 5851 VTy = llvm::VectorType::get(Int16Ty, 4); 5852 llvm::Type *Tys[2] = { Ty, VTy }; 5853 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5854 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5855 } 5856 case NEON::BI__builtin_neon_vaddlvq_s8: { 5857 Int = Intrinsic::aarch64_neon_saddlv; 5858 Ty = Int32Ty; 5859 VTy = llvm::VectorType::get(Int8Ty, 16); 5860 llvm::Type *Tys[2] = { Ty, VTy }; 5861 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5862 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5863 return Builder.CreateTrunc(Ops[0], Int16Ty); 5864 } 5865 case NEON::BI__builtin_neon_vaddlvq_s16: { 5866 Int = Intrinsic::aarch64_neon_saddlv; 5867 Ty = Int32Ty; 5868 VTy = llvm::VectorType::get(Int16Ty, 8); 5869 llvm::Type *Tys[2] = { Ty, VTy }; 5870 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5871 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5872 } 5873 case NEON::BI__builtin_neon_vsri_n_v: 5874 case NEON::BI__builtin_neon_vsriq_n_v: { 5875 Int = Intrinsic::aarch64_neon_vsri; 5876 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty); 5877 return EmitNeonCall(Intrin, Ops, "vsri_n"); 5878 } 5879 case NEON::BI__builtin_neon_vsli_n_v: 5880 case NEON::BI__builtin_neon_vsliq_n_v: { 5881 Int = Intrinsic::aarch64_neon_vsli; 5882 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty); 5883 return EmitNeonCall(Intrin, Ops, "vsli_n"); 5884 } 5885 case NEON::BI__builtin_neon_vsra_n_v: 5886 case NEON::BI__builtin_neon_vsraq_n_v: 5887 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5888 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); 5889 return Builder.CreateAdd(Ops[0], Ops[1]); 5890 case NEON::BI__builtin_neon_vrsra_n_v: 5891 case NEON::BI__builtin_neon_vrsraq_n_v: { 5892 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl; 5893 SmallVector<llvm::Value*,2> TmpOps; 5894 TmpOps.push_back(Ops[1]); 5895 TmpOps.push_back(Ops[2]); 5896 Function* F = CGM.getIntrinsic(Int, Ty); 5897 llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vrshr_n", 1, true); 5898 Ops[0] = Builder.CreateBitCast(Ops[0], VTy); 5899 return Builder.CreateAdd(Ops[0], tmp); 5900 } 5901 // FIXME: Sharing loads & stores with 32-bit is complicated by the absence 5902 // of an Align parameter here. 5903 case NEON::BI__builtin_neon_vld1_x2_v: 5904 case NEON::BI__builtin_neon_vld1q_x2_v: 5905 case NEON::BI__builtin_neon_vld1_x3_v: 5906 case NEON::BI__builtin_neon_vld1q_x3_v: 5907 case NEON::BI__builtin_neon_vld1_x4_v: 5908 case NEON::BI__builtin_neon_vld1q_x4_v: { 5909 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType()); 5910 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5911 llvm::Type *Tys[2] = { VTy, PTy }; 5912 unsigned Int; 5913 switch (BuiltinID) { 5914 case NEON::BI__builtin_neon_vld1_x2_v: 5915 case NEON::BI__builtin_neon_vld1q_x2_v: 5916 Int = Intrinsic::aarch64_neon_ld1x2; 5917 break; 5918 case NEON::BI__builtin_neon_vld1_x3_v: 5919 case NEON::BI__builtin_neon_vld1q_x3_v: 5920 Int = Intrinsic::aarch64_neon_ld1x3; 5921 break; 5922 case NEON::BI__builtin_neon_vld1_x4_v: 5923 case NEON::BI__builtin_neon_vld1q_x4_v: 5924 Int = Intrinsic::aarch64_neon_ld1x4; 5925 break; 5926 } 5927 Function *F = CGM.getIntrinsic(Int, Tys); 5928 Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN"); 5929 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 5930 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5931 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 5932 } 5933 case NEON::BI__builtin_neon_vst1_x2_v: 5934 case NEON::BI__builtin_neon_vst1q_x2_v: 5935 case NEON::BI__builtin_neon_vst1_x3_v: 5936 case NEON::BI__builtin_neon_vst1q_x3_v: 5937 case NEON::BI__builtin_neon_vst1_x4_v: 5938 case NEON::BI__builtin_neon_vst1q_x4_v: { 5939 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType()); 5940 llvm::Type *Tys[2] = { VTy, PTy }; 5941 unsigned Int; 5942 switch (BuiltinID) { 5943 case NEON::BI__builtin_neon_vst1_x2_v: 5944 case NEON::BI__builtin_neon_vst1q_x2_v: 5945 Int = Intrinsic::aarch64_neon_st1x2; 5946 break; 5947 case NEON::BI__builtin_neon_vst1_x3_v: 5948 case NEON::BI__builtin_neon_vst1q_x3_v: 5949 Int = Intrinsic::aarch64_neon_st1x3; 5950 break; 5951 case NEON::BI__builtin_neon_vst1_x4_v: 5952 case NEON::BI__builtin_neon_vst1q_x4_v: 5953 Int = Intrinsic::aarch64_neon_st1x4; 5954 break; 5955 } 5956 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); 5957 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, ""); 5958 } 5959 case NEON::BI__builtin_neon_vld1_v: 5960 case NEON::BI__builtin_neon_vld1q_v: 5961 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); 5962 return Builder.CreateDefaultAlignedLoad(Ops[0]); 5963 case NEON::BI__builtin_neon_vst1_v: 5964 case NEON::BI__builtin_neon_vst1q_v: 5965 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); 5966 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); 5967 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 5968 case NEON::BI__builtin_neon_vld1_lane_v: 5969 case NEON::BI__builtin_neon_vld1q_lane_v: 5970 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5971 Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 5972 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5973 Ops[0] = Builder.CreateDefaultAlignedLoad(Ops[0]); 5974 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane"); 5975 case NEON::BI__builtin_neon_vld1_dup_v: 5976 case NEON::BI__builtin_neon_vld1q_dup_v: { 5977 Value *V = UndefValue::get(Ty); 5978 Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 5979 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5980 Ops[0] = Builder.CreateDefaultAlignedLoad(Ops[0]); 5981 llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); 5982 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI); 5983 return EmitNeonSplat(Ops[0], CI); 5984 } 5985 case NEON::BI__builtin_neon_vst1_lane_v: 5986 case NEON::BI__builtin_neon_vst1q_lane_v: 5987 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5988 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); 5989 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 5990 return Builder.CreateDefaultAlignedStore(Ops[1], 5991 Builder.CreateBitCast(Ops[0], Ty)); 5992 case NEON::BI__builtin_neon_vld2_v: 5993 case NEON::BI__builtin_neon_vld2q_v: { 5994 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 5995 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5996 llvm::Type *Tys[2] = { VTy, PTy }; 5997 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys); 5998 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); 5999 Ops[0] = Builder.CreateBitCast(Ops[0], 6000 llvm::PointerType::getUnqual(Ops[1]->getType())); 6001 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6002 } 6003 case NEON::BI__builtin_neon_vld3_v: 6004 case NEON::BI__builtin_neon_vld3q_v: { 6005 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 6006 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6007 llvm::Type *Tys[2] = { VTy, PTy }; 6008 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys); 6009 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); 6010 Ops[0] = Builder.CreateBitCast(Ops[0], 6011 llvm::PointerType::getUnqual(Ops[1]->getType())); 6012 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6013 } 6014 case NEON::BI__builtin_neon_vld4_v: 6015 case NEON::BI__builtin_neon_vld4q_v: { 6016 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 6017 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6018 llvm::Type *Tys[2] = { VTy, PTy }; 6019 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys); 6020 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); 6021 Ops[0] = Builder.CreateBitCast(Ops[0], 6022 llvm::PointerType::getUnqual(Ops[1]->getType())); 6023 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6024 } 6025 case NEON::BI__builtin_neon_vld2_dup_v: 6026 case NEON::BI__builtin_neon_vld2q_dup_v: { 6027 llvm::Type *PTy = 6028 llvm::PointerType::getUnqual(VTy->getElementType()); 6029 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6030 llvm::Type *Tys[2] = { VTy, PTy }; 6031 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys); 6032 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); 6033 Ops[0] = Builder.CreateBitCast(Ops[0], 6034 llvm::PointerType::getUnqual(Ops[1]->getType())); 6035 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6036 } 6037 case NEON::BI__builtin_neon_vld3_dup_v: 6038 case NEON::BI__builtin_neon_vld3q_dup_v: { 6039 llvm::Type *PTy = 6040 llvm::PointerType::getUnqual(VTy->getElementType()); 6041 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6042 llvm::Type *Tys[2] = { VTy, PTy }; 6043 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys); 6044 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); 6045 Ops[0] = Builder.CreateBitCast(Ops[0], 6046 llvm::PointerType::getUnqual(Ops[1]->getType())); 6047 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6048 } 6049 case NEON::BI__builtin_neon_vld4_dup_v: 6050 case NEON::BI__builtin_neon_vld4q_dup_v: { 6051 llvm::Type *PTy = 6052 llvm::PointerType::getUnqual(VTy->getElementType()); 6053 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6054 llvm::Type *Tys[2] = { VTy, PTy }; 6055 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys); 6056 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); 6057 Ops[0] = Builder.CreateBitCast(Ops[0], 6058 llvm::PointerType::getUnqual(Ops[1]->getType())); 6059 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6060 } 6061 case NEON::BI__builtin_neon_vld2_lane_v: 6062 case NEON::BI__builtin_neon_vld2q_lane_v: { 6063 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 6064 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys); 6065 Ops.push_back(Ops[1]); 6066 Ops.erase(Ops.begin()+1); 6067 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 6068 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 6069 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); 6070 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane"); 6071 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 6072 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 6073 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6074 } 6075 case NEON::BI__builtin_neon_vld3_lane_v: 6076 case NEON::BI__builtin_neon_vld3q_lane_v: { 6077 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 6078 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys); 6079 Ops.push_back(Ops[1]); 6080 Ops.erase(Ops.begin()+1); 6081 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 6082 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 6083 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); 6084 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); 6085 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane"); 6086 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 6087 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 6088 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6089 } 6090 case NEON::BI__builtin_neon_vld4_lane_v: 6091 case NEON::BI__builtin_neon_vld4q_lane_v: { 6092 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 6093 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys); 6094 Ops.push_back(Ops[1]); 6095 Ops.erase(Ops.begin()+1); 6096 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 6097 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 6098 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); 6099 Ops[4] = Builder.CreateBitCast(Ops[4], Ty); 6100 Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty); 6101 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld4_lane"); 6102 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 6103 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 6104 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6105 } 6106 case NEON::BI__builtin_neon_vst2_v: 6107 case NEON::BI__builtin_neon_vst2q_v: { 6108 Ops.push_back(Ops[0]); 6109 Ops.erase(Ops.begin()); 6110 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() }; 6111 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys), 6112 Ops, ""); 6113 } 6114 case NEON::BI__builtin_neon_vst2_lane_v: 6115 case NEON::BI__builtin_neon_vst2q_lane_v: { 6116 Ops.push_back(Ops[0]); 6117 Ops.erase(Ops.begin()); 6118 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty); 6119 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; 6120 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys), 6121 Ops, ""); 6122 } 6123 case NEON::BI__builtin_neon_vst3_v: 6124 case NEON::BI__builtin_neon_vst3q_v: { 6125 Ops.push_back(Ops[0]); 6126 Ops.erase(Ops.begin()); 6127 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; 6128 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys), 6129 Ops, ""); 6130 } 6131 case NEON::BI__builtin_neon_vst3_lane_v: 6132 case NEON::BI__builtin_neon_vst3q_lane_v: { 6133 Ops.push_back(Ops[0]); 6134 Ops.erase(Ops.begin()); 6135 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); 6136 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; 6137 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys), 6138 Ops, ""); 6139 } 6140 case NEON::BI__builtin_neon_vst4_v: 6141 case NEON::BI__builtin_neon_vst4q_v: { 6142 Ops.push_back(Ops[0]); 6143 Ops.erase(Ops.begin()); 6144 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; 6145 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys), 6146 Ops, ""); 6147 } 6148 case NEON::BI__builtin_neon_vst4_lane_v: 6149 case NEON::BI__builtin_neon_vst4q_lane_v: { 6150 Ops.push_back(Ops[0]); 6151 Ops.erase(Ops.begin()); 6152 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); 6153 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() }; 6154 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys), 6155 Ops, ""); 6156 } 6157 case NEON::BI__builtin_neon_vtrn_v: 6158 case NEON::BI__builtin_neon_vtrnq_v: { 6159 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 6160 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 6161 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 6162 Value *SV = nullptr; 6163 6164 for (unsigned vi = 0; vi != 2; ++vi) { 6165 SmallVector<Constant*, 16> Indices; 6166 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 6167 Indices.push_back(ConstantInt::get(Int32Ty, i+vi)); 6168 Indices.push_back(ConstantInt::get(Int32Ty, i+e+vi)); 6169 } 6170 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 6171 SV = llvm::ConstantVector::get(Indices); 6172 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn"); 6173 SV = Builder.CreateDefaultAlignedStore(SV, Addr); 6174 } 6175 return SV; 6176 } 6177 case NEON::BI__builtin_neon_vuzp_v: 6178 case NEON::BI__builtin_neon_vuzpq_v: { 6179 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 6180 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 6181 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 6182 Value *SV = nullptr; 6183 6184 for (unsigned vi = 0; vi != 2; ++vi) { 6185 SmallVector<Constant*, 16> Indices; 6186 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 6187 Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi)); 6188 6189 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 6190 SV = llvm::ConstantVector::get(Indices); 6191 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp"); 6192 SV = Builder.CreateDefaultAlignedStore(SV, Addr); 6193 } 6194 return SV; 6195 } 6196 case NEON::BI__builtin_neon_vzip_v: 6197 case NEON::BI__builtin_neon_vzipq_v: { 6198 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 6199 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 6200 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 6201 Value *SV = nullptr; 6202 6203 for (unsigned vi = 0; vi != 2; ++vi) { 6204 SmallVector<Constant*, 16> Indices; 6205 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 6206 Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1)); 6207 Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e)); 6208 } 6209 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 6210 SV = llvm::ConstantVector::get(Indices); 6211 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip"); 6212 SV = Builder.CreateDefaultAlignedStore(SV, Addr); 6213 } 6214 return SV; 6215 } 6216 case NEON::BI__builtin_neon_vqtbl1q_v: { 6217 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty), 6218 Ops, "vtbl1"); 6219 } 6220 case NEON::BI__builtin_neon_vqtbl2q_v: { 6221 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty), 6222 Ops, "vtbl2"); 6223 } 6224 case NEON::BI__builtin_neon_vqtbl3q_v: { 6225 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty), 6226 Ops, "vtbl3"); 6227 } 6228 case NEON::BI__builtin_neon_vqtbl4q_v: { 6229 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty), 6230 Ops, "vtbl4"); 6231 } 6232 case NEON::BI__builtin_neon_vqtbx1q_v: { 6233 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty), 6234 Ops, "vtbx1"); 6235 } 6236 case NEON::BI__builtin_neon_vqtbx2q_v: { 6237 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty), 6238 Ops, "vtbx2"); 6239 } 6240 case NEON::BI__builtin_neon_vqtbx3q_v: { 6241 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty), 6242 Ops, "vtbx3"); 6243 } 6244 case NEON::BI__builtin_neon_vqtbx4q_v: { 6245 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty), 6246 Ops, "vtbx4"); 6247 } 6248 case NEON::BI__builtin_neon_vsqadd_v: 6249 case NEON::BI__builtin_neon_vsqaddq_v: { 6250 Int = Intrinsic::aarch64_neon_usqadd; 6251 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd"); 6252 } 6253 case NEON::BI__builtin_neon_vuqadd_v: 6254 case NEON::BI__builtin_neon_vuqaddq_v: { 6255 Int = Intrinsic::aarch64_neon_suqadd; 6256 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd"); 6257 } 6258 } 6259 } 6260 6261 llvm::Value *CodeGenFunction:: 6262 BuildVector(ArrayRef<llvm::Value*> Ops) { 6263 assert((Ops.size() & (Ops.size() - 1)) == 0 && 6264 "Not a power-of-two sized vector!"); 6265 bool AllConstants = true; 6266 for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i) 6267 AllConstants &= isa<Constant>(Ops[i]); 6268 6269 // If this is a constant vector, create a ConstantVector. 6270 if (AllConstants) { 6271 SmallVector<llvm::Constant*, 16> CstOps; 6272 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 6273 CstOps.push_back(cast<Constant>(Ops[i])); 6274 return llvm::ConstantVector::get(CstOps); 6275 } 6276 6277 // Otherwise, insertelement the values to build the vector. 6278 Value *Result = 6279 llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size())); 6280 6281 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 6282 Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i)); 6283 6284 return Result; 6285 } 6286 6287 Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, 6288 const CallExpr *E) { 6289 if (BuiltinID == X86::BI__builtin_ms_va_start || 6290 BuiltinID == X86::BI__builtin_ms_va_end) 6291 return EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).getPointer(), 6292 BuiltinID == X86::BI__builtin_ms_va_start); 6293 if (BuiltinID == X86::BI__builtin_ms_va_copy) { 6294 // Lower this manually. We can't reliably determine whether or not any 6295 // given va_copy() is for a Win64 va_list from the calling convention 6296 // alone, because it's legal to do this from a System V ABI function. 6297 // With opaque pointer types, we won't have enough information in LLVM 6298 // IR to determine this from the argument types, either. Best to do it 6299 // now, while we have enough information. 6300 Address DestAddr = EmitMSVAListRef(E->getArg(0)); 6301 Address SrcAddr = EmitMSVAListRef(E->getArg(1)); 6302 6303 llvm::Type *BPP = Int8PtrPtrTy; 6304 6305 DestAddr = Address(Builder.CreateBitCast(DestAddr.getPointer(), BPP, "cp"), 6306 DestAddr.getAlignment()); 6307 SrcAddr = Address(Builder.CreateBitCast(SrcAddr.getPointer(), BPP, "ap"), 6308 SrcAddr.getAlignment()); 6309 6310 Value *ArgPtr = Builder.CreateLoad(SrcAddr, "ap.val"); 6311 return Builder.CreateStore(ArgPtr, DestAddr); 6312 } 6313 6314 SmallVector<Value*, 4> Ops; 6315 6316 // Find out if any arguments are required to be integer constant expressions. 6317 unsigned ICEArguments = 0; 6318 ASTContext::GetBuiltinTypeError Error; 6319 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 6320 assert(Error == ASTContext::GE_None && "Should not codegen an error"); 6321 6322 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) { 6323 // If this is a normal argument, just emit it as a scalar. 6324 if ((ICEArguments & (1 << i)) == 0) { 6325 Ops.push_back(EmitScalarExpr(E->getArg(i))); 6326 continue; 6327 } 6328 6329 // If this is required to be a constant, constant fold it so that we know 6330 // that the generated intrinsic gets a ConstantInt. 6331 llvm::APSInt Result; 6332 bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext()); 6333 assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst; 6334 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); 6335 } 6336 6337 switch (BuiltinID) { 6338 default: return nullptr; 6339 case X86::BI__builtin_cpu_supports: { 6340 const Expr *FeatureExpr = E->getArg(0)->IgnoreParenCasts(); 6341 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString(); 6342 6343 // TODO: When/if this becomes more than x86 specific then use a TargetInfo 6344 // based mapping. 6345 // Processor features and mapping to processor feature value. 6346 enum X86Features { 6347 CMOV = 0, 6348 MMX, 6349 POPCNT, 6350 SSE, 6351 SSE2, 6352 SSE3, 6353 SSSE3, 6354 SSE4_1, 6355 SSE4_2, 6356 AVX, 6357 AVX2, 6358 SSE4_A, 6359 FMA4, 6360 XOP, 6361 FMA, 6362 AVX512F, 6363 BMI, 6364 BMI2, 6365 AES, 6366 PCLMUL, 6367 AVX512VL, 6368 AVX512BW, 6369 AVX512DQ, 6370 AVX512CD, 6371 AVX512ER, 6372 AVX512PF, 6373 AVX512VBMI, 6374 AVX512IFMA, 6375 MAX 6376 }; 6377 6378 X86Features Feature = StringSwitch<X86Features>(FeatureStr) 6379 .Case("cmov", X86Features::CMOV) 6380 .Case("mmx", X86Features::MMX) 6381 .Case("popcnt", X86Features::POPCNT) 6382 .Case("sse", X86Features::SSE) 6383 .Case("sse2", X86Features::SSE2) 6384 .Case("sse3", X86Features::SSE3) 6385 .Case("ssse3", X86Features::SSSE3) 6386 .Case("sse4.1", X86Features::SSE4_1) 6387 .Case("sse4.2", X86Features::SSE4_2) 6388 .Case("avx", X86Features::AVX) 6389 .Case("avx2", X86Features::AVX2) 6390 .Case("sse4a", X86Features::SSE4_A) 6391 .Case("fma4", X86Features::FMA4) 6392 .Case("xop", X86Features::XOP) 6393 .Case("fma", X86Features::FMA) 6394 .Case("avx512f", X86Features::AVX512F) 6395 .Case("bmi", X86Features::BMI) 6396 .Case("bmi2", X86Features::BMI2) 6397 .Case("aes", X86Features::AES) 6398 .Case("pclmul", X86Features::PCLMUL) 6399 .Case("avx512vl", X86Features::AVX512VL) 6400 .Case("avx512bw", X86Features::AVX512BW) 6401 .Case("avx512dq", X86Features::AVX512DQ) 6402 .Case("avx512cd", X86Features::AVX512CD) 6403 .Case("avx512er", X86Features::AVX512ER) 6404 .Case("avx512pf", X86Features::AVX512PF) 6405 .Case("avx512vbmi", X86Features::AVX512VBMI) 6406 .Case("avx512ifma", X86Features::AVX512IFMA) 6407 .Default(X86Features::MAX); 6408 assert(Feature != X86Features::MAX && "Invalid feature!"); 6409 6410 // Matching the struct layout from the compiler-rt/libgcc structure that is 6411 // filled in: 6412 // unsigned int __cpu_vendor; 6413 // unsigned int __cpu_type; 6414 // unsigned int __cpu_subtype; 6415 // unsigned int __cpu_features[1]; 6416 llvm::Type *STy = llvm::StructType::get( 6417 Int32Ty, Int32Ty, Int32Ty, llvm::ArrayType::get(Int32Ty, 1), nullptr); 6418 6419 // Grab the global __cpu_model. 6420 llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy, "__cpu_model"); 6421 6422 // Grab the first (0th) element from the field __cpu_features off of the 6423 // global in the struct STy. 6424 Value *Idxs[] = { 6425 ConstantInt::get(Int32Ty, 0), 6426 ConstantInt::get(Int32Ty, 3), 6427 ConstantInt::get(Int32Ty, 0) 6428 }; 6429 Value *CpuFeatures = Builder.CreateGEP(STy, CpuModel, Idxs); 6430 Value *Features = Builder.CreateAlignedLoad(CpuFeatures, 6431 CharUnits::fromQuantity(4)); 6432 6433 // Check the value of the bit corresponding to the feature requested. 6434 Value *Bitset = Builder.CreateAnd( 6435 Features, llvm::ConstantInt::get(Int32Ty, 1ULL << Feature)); 6436 return Builder.CreateICmpNE(Bitset, llvm::ConstantInt::get(Int32Ty, 0)); 6437 } 6438 case X86::BI_mm_prefetch: { 6439 Value *Address = Ops[0]; 6440 Value *RW = ConstantInt::get(Int32Ty, 0); 6441 Value *Locality = Ops[1]; 6442 Value *Data = ConstantInt::get(Int32Ty, 1); 6443 Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 6444 return Builder.CreateCall(F, {Address, RW, Locality, Data}); 6445 } 6446 case X86::BI__builtin_ia32_undef128: 6447 case X86::BI__builtin_ia32_undef256: 6448 case X86::BI__builtin_ia32_undef512: 6449 return UndefValue::get(ConvertType(E->getType())); 6450 case X86::BI__builtin_ia32_vec_init_v8qi: 6451 case X86::BI__builtin_ia32_vec_init_v4hi: 6452 case X86::BI__builtin_ia32_vec_init_v2si: 6453 return Builder.CreateBitCast(BuildVector(Ops), 6454 llvm::Type::getX86_MMXTy(getLLVMContext())); 6455 case X86::BI__builtin_ia32_vec_ext_v2si: 6456 return Builder.CreateExtractElement(Ops[0], 6457 llvm::ConstantInt::get(Ops[1]->getType(), 0)); 6458 case X86::BI__builtin_ia32_ldmxcsr: { 6459 Address Tmp = CreateMemTemp(E->getArg(0)->getType()); 6460 Builder.CreateStore(Ops[0], Tmp); 6461 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr), 6462 Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy)); 6463 } 6464 case X86::BI__builtin_ia32_stmxcsr: { 6465 Address Tmp = CreateMemTemp(E->getType()); 6466 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr), 6467 Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy)); 6468 return Builder.CreateLoad(Tmp, "stmxcsr"); 6469 } 6470 case X86::BI__builtin_ia32_xsave: 6471 case X86::BI__builtin_ia32_xsave64: 6472 case X86::BI__builtin_ia32_xrstor: 6473 case X86::BI__builtin_ia32_xrstor64: 6474 case X86::BI__builtin_ia32_xsaveopt: 6475 case X86::BI__builtin_ia32_xsaveopt64: 6476 case X86::BI__builtin_ia32_xrstors: 6477 case X86::BI__builtin_ia32_xrstors64: 6478 case X86::BI__builtin_ia32_xsavec: 6479 case X86::BI__builtin_ia32_xsavec64: 6480 case X86::BI__builtin_ia32_xsaves: 6481 case X86::BI__builtin_ia32_xsaves64: { 6482 Intrinsic::ID ID; 6483 #define INTRINSIC_X86_XSAVE_ID(NAME) \ 6484 case X86::BI__builtin_ia32_##NAME: \ 6485 ID = Intrinsic::x86_##NAME; \ 6486 break 6487 switch (BuiltinID) { 6488 default: llvm_unreachable("Unsupported intrinsic!"); 6489 INTRINSIC_X86_XSAVE_ID(xsave); 6490 INTRINSIC_X86_XSAVE_ID(xsave64); 6491 INTRINSIC_X86_XSAVE_ID(xrstor); 6492 INTRINSIC_X86_XSAVE_ID(xrstor64); 6493 INTRINSIC_X86_XSAVE_ID(xsaveopt); 6494 INTRINSIC_X86_XSAVE_ID(xsaveopt64); 6495 INTRINSIC_X86_XSAVE_ID(xrstors); 6496 INTRINSIC_X86_XSAVE_ID(xrstors64); 6497 INTRINSIC_X86_XSAVE_ID(xsavec); 6498 INTRINSIC_X86_XSAVE_ID(xsavec64); 6499 INTRINSIC_X86_XSAVE_ID(xsaves); 6500 INTRINSIC_X86_XSAVE_ID(xsaves64); 6501 } 6502 #undef INTRINSIC_X86_XSAVE_ID 6503 Value *Mhi = Builder.CreateTrunc( 6504 Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty); 6505 Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty); 6506 Ops[1] = Mhi; 6507 Ops.push_back(Mlo); 6508 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); 6509 } 6510 case X86::BI__builtin_ia32_storehps: 6511 case X86::BI__builtin_ia32_storelps: { 6512 llvm::Type *PtrTy = llvm::PointerType::getUnqual(Int64Ty); 6513 llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2); 6514 6515 // cast val v2i64 6516 Ops[1] = Builder.CreateBitCast(Ops[1], VecTy, "cast"); 6517 6518 // extract (0, 1) 6519 unsigned Index = BuiltinID == X86::BI__builtin_ia32_storelps ? 0 : 1; 6520 llvm::Value *Idx = llvm::ConstantInt::get(SizeTy, Index); 6521 Ops[1] = Builder.CreateExtractElement(Ops[1], Idx, "extract"); 6522 6523 // cast pointer to i64 & store 6524 Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy); 6525 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6526 } 6527 case X86::BI__builtin_ia32_palignr128: 6528 case X86::BI__builtin_ia32_palignr256: { 6529 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); 6530 6531 unsigned NumElts = 6532 cast<llvm::VectorType>(Ops[0]->getType())->getNumElements(); 6533 assert(NumElts % 16 == 0); 6534 unsigned NumLanes = NumElts / 16; 6535 unsigned NumLaneElts = NumElts / NumLanes; 6536 6537 // If palignr is shifting the pair of vectors more than the size of two 6538 // lanes, emit zero. 6539 if (ShiftVal >= (2 * NumLaneElts)) 6540 return llvm::Constant::getNullValue(ConvertType(E->getType())); 6541 6542 // If palignr is shifting the pair of input vectors more than one lane, 6543 // but less than two lanes, convert to shifting in zeroes. 6544 if (ShiftVal > NumLaneElts) { 6545 ShiftVal -= NumLaneElts; 6546 Ops[1] = Ops[0]; 6547 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType()); 6548 } 6549 6550 uint32_t Indices[32]; 6551 // 256-bit palignr operates on 128-bit lanes so we need to handle that 6552 for (unsigned l = 0; l != NumElts; l += NumLaneElts) { 6553 for (unsigned i = 0; i != NumLaneElts; ++i) { 6554 unsigned Idx = ShiftVal + i; 6555 if (Idx >= NumLaneElts) 6556 Idx += NumElts - NumLaneElts; // End of lane, switch operand. 6557 Indices[l + i] = Idx + l; 6558 } 6559 } 6560 6561 Value *SV = llvm::ConstantDataVector::get(getLLVMContext(), 6562 makeArrayRef(Indices, NumElts)); 6563 return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); 6564 } 6565 case X86::BI__builtin_ia32_pslldqi256: { 6566 // Shift value is in bits so divide by 8. 6567 unsigned shiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() >> 3; 6568 6569 // If pslldq is shifting the vector more than 15 bytes, emit zero. 6570 if (shiftVal >= 16) 6571 return llvm::Constant::getNullValue(ConvertType(E->getType())); 6572 6573 uint32_t Indices[32]; 6574 // 256-bit pslldq operates on 128-bit lanes so we need to handle that 6575 for (unsigned l = 0; l != 32; l += 16) { 6576 for (unsigned i = 0; i != 16; ++i) { 6577 unsigned Idx = 32 + i - shiftVal; 6578 if (Idx < 32) Idx -= 16; // end of lane, switch operand. 6579 Indices[l + i] = Idx + l; 6580 } 6581 } 6582 6583 llvm::Type *VecTy = llvm::VectorType::get(Int8Ty, 32); 6584 Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); 6585 Value *Zero = llvm::Constant::getNullValue(VecTy); 6586 6587 Value *SV = llvm::ConstantDataVector::get(getLLVMContext(), Indices); 6588 SV = Builder.CreateShuffleVector(Zero, Ops[0], SV, "pslldq"); 6589 llvm::Type *ResultType = ConvertType(E->getType()); 6590 return Builder.CreateBitCast(SV, ResultType, "cast"); 6591 } 6592 case X86::BI__builtin_ia32_psrldqi256: { 6593 // Shift value is in bits so divide by 8. 6594 unsigned shiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() >> 3; 6595 6596 // If psrldq is shifting the vector more than 15 bytes, emit zero. 6597 if (shiftVal >= 16) 6598 return llvm::Constant::getNullValue(ConvertType(E->getType())); 6599 6600 uint32_t Indices[32]; 6601 // 256-bit psrldq operates on 128-bit lanes so we need to handle that 6602 for (unsigned l = 0; l != 32; l += 16) { 6603 for (unsigned i = 0; i != 16; ++i) { 6604 unsigned Idx = i + shiftVal; 6605 if (Idx >= 16) Idx += 16; // end of lane, switch operand. 6606 Indices[l + i] = Idx + l; 6607 } 6608 } 6609 6610 llvm::Type *VecTy = llvm::VectorType::get(Int8Ty, 32); 6611 Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); 6612 Value *Zero = llvm::Constant::getNullValue(VecTy); 6613 6614 Value *SV = llvm::ConstantDataVector::get(getLLVMContext(), Indices); 6615 SV = Builder.CreateShuffleVector(Ops[0], Zero, SV, "psrldq"); 6616 llvm::Type *ResultType = ConvertType(E->getType()); 6617 return Builder.CreateBitCast(SV, ResultType, "cast"); 6618 } 6619 case X86::BI__builtin_ia32_movntps: 6620 case X86::BI__builtin_ia32_movntps256: 6621 case X86::BI__builtin_ia32_movntpd: 6622 case X86::BI__builtin_ia32_movntpd256: 6623 case X86::BI__builtin_ia32_movntdq: 6624 case X86::BI__builtin_ia32_movntdq256: 6625 case X86::BI__builtin_ia32_movnti: 6626 case X86::BI__builtin_ia32_movnti64: { 6627 llvm::MDNode *Node = llvm::MDNode::get( 6628 getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1))); 6629 6630 // Convert the type of the pointer to a pointer to the stored type. 6631 Value *BC = Builder.CreateBitCast(Ops[0], 6632 llvm::PointerType::getUnqual(Ops[1]->getType()), 6633 "cast"); 6634 StoreInst *SI = Builder.CreateDefaultAlignedStore(Ops[1], BC); 6635 SI->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node); 6636 6637 // If the operand is an integer, we can't assume alignment. Otherwise, 6638 // assume natural alignment. 6639 QualType ArgTy = E->getArg(1)->getType(); 6640 unsigned Align; 6641 if (ArgTy->isIntegerType()) 6642 Align = 1; 6643 else 6644 Align = getContext().getTypeSizeInChars(ArgTy).getQuantity(); 6645 SI->setAlignment(Align); 6646 return SI; 6647 } 6648 // 3DNow! 6649 case X86::BI__builtin_ia32_pswapdsf: 6650 case X86::BI__builtin_ia32_pswapdsi: { 6651 llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext()); 6652 Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast"); 6653 llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_3dnowa_pswapd); 6654 return Builder.CreateCall(F, Ops, "pswapd"); 6655 } 6656 case X86::BI__builtin_ia32_rdrand16_step: 6657 case X86::BI__builtin_ia32_rdrand32_step: 6658 case X86::BI__builtin_ia32_rdrand64_step: 6659 case X86::BI__builtin_ia32_rdseed16_step: 6660 case X86::BI__builtin_ia32_rdseed32_step: 6661 case X86::BI__builtin_ia32_rdseed64_step: { 6662 Intrinsic::ID ID; 6663 switch (BuiltinID) { 6664 default: llvm_unreachable("Unsupported intrinsic!"); 6665 case X86::BI__builtin_ia32_rdrand16_step: 6666 ID = Intrinsic::x86_rdrand_16; 6667 break; 6668 case X86::BI__builtin_ia32_rdrand32_step: 6669 ID = Intrinsic::x86_rdrand_32; 6670 break; 6671 case X86::BI__builtin_ia32_rdrand64_step: 6672 ID = Intrinsic::x86_rdrand_64; 6673 break; 6674 case X86::BI__builtin_ia32_rdseed16_step: 6675 ID = Intrinsic::x86_rdseed_16; 6676 break; 6677 case X86::BI__builtin_ia32_rdseed32_step: 6678 ID = Intrinsic::x86_rdseed_32; 6679 break; 6680 case X86::BI__builtin_ia32_rdseed64_step: 6681 ID = Intrinsic::x86_rdseed_64; 6682 break; 6683 } 6684 6685 Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID)); 6686 Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 0), 6687 Ops[0]); 6688 return Builder.CreateExtractValue(Call, 1); 6689 } 6690 // SSE comparison intrisics 6691 case X86::BI__builtin_ia32_cmpeqps: 6692 case X86::BI__builtin_ia32_cmpltps: 6693 case X86::BI__builtin_ia32_cmpleps: 6694 case X86::BI__builtin_ia32_cmpunordps: 6695 case X86::BI__builtin_ia32_cmpneqps: 6696 case X86::BI__builtin_ia32_cmpnltps: 6697 case X86::BI__builtin_ia32_cmpnleps: 6698 case X86::BI__builtin_ia32_cmpordps: 6699 case X86::BI__builtin_ia32_cmpeqss: 6700 case X86::BI__builtin_ia32_cmpltss: 6701 case X86::BI__builtin_ia32_cmpless: 6702 case X86::BI__builtin_ia32_cmpunordss: 6703 case X86::BI__builtin_ia32_cmpneqss: 6704 case X86::BI__builtin_ia32_cmpnltss: 6705 case X86::BI__builtin_ia32_cmpnless: 6706 case X86::BI__builtin_ia32_cmpordss: 6707 case X86::BI__builtin_ia32_cmpeqpd: 6708 case X86::BI__builtin_ia32_cmpltpd: 6709 case X86::BI__builtin_ia32_cmplepd: 6710 case X86::BI__builtin_ia32_cmpunordpd: 6711 case X86::BI__builtin_ia32_cmpneqpd: 6712 case X86::BI__builtin_ia32_cmpnltpd: 6713 case X86::BI__builtin_ia32_cmpnlepd: 6714 case X86::BI__builtin_ia32_cmpordpd: 6715 case X86::BI__builtin_ia32_cmpeqsd: 6716 case X86::BI__builtin_ia32_cmpltsd: 6717 case X86::BI__builtin_ia32_cmplesd: 6718 case X86::BI__builtin_ia32_cmpunordsd: 6719 case X86::BI__builtin_ia32_cmpneqsd: 6720 case X86::BI__builtin_ia32_cmpnltsd: 6721 case X86::BI__builtin_ia32_cmpnlesd: 6722 case X86::BI__builtin_ia32_cmpordsd: 6723 // These exist so that the builtin that takes an immediate can be bounds 6724 // checked by clang to avoid passing bad immediates to the backend. Since 6725 // AVX has a larger immediate than SSE we would need separate builtins to 6726 // do the different bounds checking. Rather than create a clang specific 6727 // SSE only builtin, this implements eight separate builtins to match gcc 6728 // implementation. 6729 6730 // Choose the immediate. 6731 unsigned Imm; 6732 switch (BuiltinID) { 6733 default: llvm_unreachable("Unsupported intrinsic!"); 6734 case X86::BI__builtin_ia32_cmpeqps: 6735 case X86::BI__builtin_ia32_cmpeqss: 6736 case X86::BI__builtin_ia32_cmpeqpd: 6737 case X86::BI__builtin_ia32_cmpeqsd: 6738 Imm = 0; 6739 break; 6740 case X86::BI__builtin_ia32_cmpltps: 6741 case X86::BI__builtin_ia32_cmpltss: 6742 case X86::BI__builtin_ia32_cmpltpd: 6743 case X86::BI__builtin_ia32_cmpltsd: 6744 Imm = 1; 6745 break; 6746 case X86::BI__builtin_ia32_cmpleps: 6747 case X86::BI__builtin_ia32_cmpless: 6748 case X86::BI__builtin_ia32_cmplepd: 6749 case X86::BI__builtin_ia32_cmplesd: 6750 Imm = 2; 6751 break; 6752 case X86::BI__builtin_ia32_cmpunordps: 6753 case X86::BI__builtin_ia32_cmpunordss: 6754 case X86::BI__builtin_ia32_cmpunordpd: 6755 case X86::BI__builtin_ia32_cmpunordsd: 6756 Imm = 3; 6757 break; 6758 case X86::BI__builtin_ia32_cmpneqps: 6759 case X86::BI__builtin_ia32_cmpneqss: 6760 case X86::BI__builtin_ia32_cmpneqpd: 6761 case X86::BI__builtin_ia32_cmpneqsd: 6762 Imm = 4; 6763 break; 6764 case X86::BI__builtin_ia32_cmpnltps: 6765 case X86::BI__builtin_ia32_cmpnltss: 6766 case X86::BI__builtin_ia32_cmpnltpd: 6767 case X86::BI__builtin_ia32_cmpnltsd: 6768 Imm = 5; 6769 break; 6770 case X86::BI__builtin_ia32_cmpnleps: 6771 case X86::BI__builtin_ia32_cmpnless: 6772 case X86::BI__builtin_ia32_cmpnlepd: 6773 case X86::BI__builtin_ia32_cmpnlesd: 6774 Imm = 6; 6775 break; 6776 case X86::BI__builtin_ia32_cmpordps: 6777 case X86::BI__builtin_ia32_cmpordss: 6778 case X86::BI__builtin_ia32_cmpordpd: 6779 case X86::BI__builtin_ia32_cmpordsd: 6780 Imm = 7; 6781 break; 6782 } 6783 6784 // Choose the intrinsic ID. 6785 const char *name; 6786 Intrinsic::ID ID; 6787 switch (BuiltinID) { 6788 default: llvm_unreachable("Unsupported intrinsic!"); 6789 case X86::BI__builtin_ia32_cmpeqps: 6790 case X86::BI__builtin_ia32_cmpltps: 6791 case X86::BI__builtin_ia32_cmpleps: 6792 case X86::BI__builtin_ia32_cmpunordps: 6793 case X86::BI__builtin_ia32_cmpneqps: 6794 case X86::BI__builtin_ia32_cmpnltps: 6795 case X86::BI__builtin_ia32_cmpnleps: 6796 case X86::BI__builtin_ia32_cmpordps: 6797 name = "cmpps"; 6798 ID = Intrinsic::x86_sse_cmp_ps; 6799 break; 6800 case X86::BI__builtin_ia32_cmpeqss: 6801 case X86::BI__builtin_ia32_cmpltss: 6802 case X86::BI__builtin_ia32_cmpless: 6803 case X86::BI__builtin_ia32_cmpunordss: 6804 case X86::BI__builtin_ia32_cmpneqss: 6805 case X86::BI__builtin_ia32_cmpnltss: 6806 case X86::BI__builtin_ia32_cmpnless: 6807 case X86::BI__builtin_ia32_cmpordss: 6808 name = "cmpss"; 6809 ID = Intrinsic::x86_sse_cmp_ss; 6810 break; 6811 case X86::BI__builtin_ia32_cmpeqpd: 6812 case X86::BI__builtin_ia32_cmpltpd: 6813 case X86::BI__builtin_ia32_cmplepd: 6814 case X86::BI__builtin_ia32_cmpunordpd: 6815 case X86::BI__builtin_ia32_cmpneqpd: 6816 case X86::BI__builtin_ia32_cmpnltpd: 6817 case X86::BI__builtin_ia32_cmpnlepd: 6818 case X86::BI__builtin_ia32_cmpordpd: 6819 name = "cmppd"; 6820 ID = Intrinsic::x86_sse2_cmp_pd; 6821 break; 6822 case X86::BI__builtin_ia32_cmpeqsd: 6823 case X86::BI__builtin_ia32_cmpltsd: 6824 case X86::BI__builtin_ia32_cmplesd: 6825 case X86::BI__builtin_ia32_cmpunordsd: 6826 case X86::BI__builtin_ia32_cmpneqsd: 6827 case X86::BI__builtin_ia32_cmpnltsd: 6828 case X86::BI__builtin_ia32_cmpnlesd: 6829 case X86::BI__builtin_ia32_cmpordsd: 6830 name = "cmpsd"; 6831 ID = Intrinsic::x86_sse2_cmp_sd; 6832 break; 6833 } 6834 6835 Ops.push_back(llvm::ConstantInt::get(Int8Ty, Imm)); 6836 llvm::Function *F = CGM.getIntrinsic(ID); 6837 return Builder.CreateCall(F, Ops, name); 6838 } 6839 } 6840 6841 6842 Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, 6843 const CallExpr *E) { 6844 SmallVector<Value*, 4> Ops; 6845 6846 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) 6847 Ops.push_back(EmitScalarExpr(E->getArg(i))); 6848 6849 Intrinsic::ID ID = Intrinsic::not_intrinsic; 6850 6851 switch (BuiltinID) { 6852 default: return nullptr; 6853 6854 // __builtin_ppc_get_timebase is GCC 4.8+'s PowerPC-specific name for what we 6855 // call __builtin_readcyclecounter. 6856 case PPC::BI__builtin_ppc_get_timebase: 6857 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::readcyclecounter)); 6858 6859 // vec_ld, vec_lvsl, vec_lvsr 6860 case PPC::BI__builtin_altivec_lvx: 6861 case PPC::BI__builtin_altivec_lvxl: 6862 case PPC::BI__builtin_altivec_lvebx: 6863 case PPC::BI__builtin_altivec_lvehx: 6864 case PPC::BI__builtin_altivec_lvewx: 6865 case PPC::BI__builtin_altivec_lvsl: 6866 case PPC::BI__builtin_altivec_lvsr: 6867 case PPC::BI__builtin_vsx_lxvd2x: 6868 case PPC::BI__builtin_vsx_lxvw4x: 6869 { 6870 Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); 6871 6872 Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]); 6873 Ops.pop_back(); 6874 6875 switch (BuiltinID) { 6876 default: llvm_unreachable("Unsupported ld/lvsl/lvsr intrinsic!"); 6877 case PPC::BI__builtin_altivec_lvx: 6878 ID = Intrinsic::ppc_altivec_lvx; 6879 break; 6880 case PPC::BI__builtin_altivec_lvxl: 6881 ID = Intrinsic::ppc_altivec_lvxl; 6882 break; 6883 case PPC::BI__builtin_altivec_lvebx: 6884 ID = Intrinsic::ppc_altivec_lvebx; 6885 break; 6886 case PPC::BI__builtin_altivec_lvehx: 6887 ID = Intrinsic::ppc_altivec_lvehx; 6888 break; 6889 case PPC::BI__builtin_altivec_lvewx: 6890 ID = Intrinsic::ppc_altivec_lvewx; 6891 break; 6892 case PPC::BI__builtin_altivec_lvsl: 6893 ID = Intrinsic::ppc_altivec_lvsl; 6894 break; 6895 case PPC::BI__builtin_altivec_lvsr: 6896 ID = Intrinsic::ppc_altivec_lvsr; 6897 break; 6898 case PPC::BI__builtin_vsx_lxvd2x: 6899 ID = Intrinsic::ppc_vsx_lxvd2x; 6900 break; 6901 case PPC::BI__builtin_vsx_lxvw4x: 6902 ID = Intrinsic::ppc_vsx_lxvw4x; 6903 break; 6904 } 6905 llvm::Function *F = CGM.getIntrinsic(ID); 6906 return Builder.CreateCall(F, Ops, ""); 6907 } 6908 6909 // vec_st 6910 case PPC::BI__builtin_altivec_stvx: 6911 case PPC::BI__builtin_altivec_stvxl: 6912 case PPC::BI__builtin_altivec_stvebx: 6913 case PPC::BI__builtin_altivec_stvehx: 6914 case PPC::BI__builtin_altivec_stvewx: 6915 case PPC::BI__builtin_vsx_stxvd2x: 6916 case PPC::BI__builtin_vsx_stxvw4x: 6917 { 6918 Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy); 6919 Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]); 6920 Ops.pop_back(); 6921 6922 switch (BuiltinID) { 6923 default: llvm_unreachable("Unsupported st intrinsic!"); 6924 case PPC::BI__builtin_altivec_stvx: 6925 ID = Intrinsic::ppc_altivec_stvx; 6926 break; 6927 case PPC::BI__builtin_altivec_stvxl: 6928 ID = Intrinsic::ppc_altivec_stvxl; 6929 break; 6930 case PPC::BI__builtin_altivec_stvebx: 6931 ID = Intrinsic::ppc_altivec_stvebx; 6932 break; 6933 case PPC::BI__builtin_altivec_stvehx: 6934 ID = Intrinsic::ppc_altivec_stvehx; 6935 break; 6936 case PPC::BI__builtin_altivec_stvewx: 6937 ID = Intrinsic::ppc_altivec_stvewx; 6938 break; 6939 case PPC::BI__builtin_vsx_stxvd2x: 6940 ID = Intrinsic::ppc_vsx_stxvd2x; 6941 break; 6942 case PPC::BI__builtin_vsx_stxvw4x: 6943 ID = Intrinsic::ppc_vsx_stxvw4x; 6944 break; 6945 } 6946 llvm::Function *F = CGM.getIntrinsic(ID); 6947 return Builder.CreateCall(F, Ops, ""); 6948 } 6949 // Square root 6950 case PPC::BI__builtin_vsx_xvsqrtsp: 6951 case PPC::BI__builtin_vsx_xvsqrtdp: { 6952 llvm::Type *ResultType = ConvertType(E->getType()); 6953 Value *X = EmitScalarExpr(E->getArg(0)); 6954 ID = Intrinsic::sqrt; 6955 llvm::Function *F = CGM.getIntrinsic(ID, ResultType); 6956 return Builder.CreateCall(F, X); 6957 } 6958 // Count leading zeros 6959 case PPC::BI__builtin_altivec_vclzb: 6960 case PPC::BI__builtin_altivec_vclzh: 6961 case PPC::BI__builtin_altivec_vclzw: 6962 case PPC::BI__builtin_altivec_vclzd: { 6963 llvm::Type *ResultType = ConvertType(E->getType()); 6964 Value *X = EmitScalarExpr(E->getArg(0)); 6965 Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false); 6966 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType); 6967 return Builder.CreateCall(F, {X, Undef}); 6968 } 6969 // Copy sign 6970 case PPC::BI__builtin_vsx_xvcpsgnsp: 6971 case PPC::BI__builtin_vsx_xvcpsgndp: { 6972 llvm::Type *ResultType = ConvertType(E->getType()); 6973 Value *X = EmitScalarExpr(E->getArg(0)); 6974 Value *Y = EmitScalarExpr(E->getArg(1)); 6975 ID = Intrinsic::copysign; 6976 llvm::Function *F = CGM.getIntrinsic(ID, ResultType); 6977 return Builder.CreateCall(F, {X, Y}); 6978 } 6979 // Rounding/truncation 6980 case PPC::BI__builtin_vsx_xvrspip: 6981 case PPC::BI__builtin_vsx_xvrdpip: 6982 case PPC::BI__builtin_vsx_xvrdpim: 6983 case PPC::BI__builtin_vsx_xvrspim: 6984 case PPC::BI__builtin_vsx_xvrdpi: 6985 case PPC::BI__builtin_vsx_xvrspi: 6986 case PPC::BI__builtin_vsx_xvrdpic: 6987 case PPC::BI__builtin_vsx_xvrspic: 6988 case PPC::BI__builtin_vsx_xvrdpiz: 6989 case PPC::BI__builtin_vsx_xvrspiz: { 6990 llvm::Type *ResultType = ConvertType(E->getType()); 6991 Value *X = EmitScalarExpr(E->getArg(0)); 6992 if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim || 6993 BuiltinID == PPC::BI__builtin_vsx_xvrspim) 6994 ID = Intrinsic::floor; 6995 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi || 6996 BuiltinID == PPC::BI__builtin_vsx_xvrspi) 6997 ID = Intrinsic::round; 6998 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic || 6999 BuiltinID == PPC::BI__builtin_vsx_xvrspic) 7000 ID = Intrinsic::nearbyint; 7001 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip || 7002 BuiltinID == PPC::BI__builtin_vsx_xvrspip) 7003 ID = Intrinsic::ceil; 7004 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz || 7005 BuiltinID == PPC::BI__builtin_vsx_xvrspiz) 7006 ID = Intrinsic::trunc; 7007 llvm::Function *F = CGM.getIntrinsic(ID, ResultType); 7008 return Builder.CreateCall(F, X); 7009 } 7010 7011 // Absolute value 7012 case PPC::BI__builtin_vsx_xvabsdp: 7013 case PPC::BI__builtin_vsx_xvabssp: { 7014 llvm::Type *ResultType = ConvertType(E->getType()); 7015 Value *X = EmitScalarExpr(E->getArg(0)); 7016 llvm::Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType); 7017 return Builder.CreateCall(F, X); 7018 } 7019 7020 // FMA variations 7021 case PPC::BI__builtin_vsx_xvmaddadp: 7022 case PPC::BI__builtin_vsx_xvmaddasp: 7023 case PPC::BI__builtin_vsx_xvnmaddadp: 7024 case PPC::BI__builtin_vsx_xvnmaddasp: 7025 case PPC::BI__builtin_vsx_xvmsubadp: 7026 case PPC::BI__builtin_vsx_xvmsubasp: 7027 case PPC::BI__builtin_vsx_xvnmsubadp: 7028 case PPC::BI__builtin_vsx_xvnmsubasp: { 7029 llvm::Type *ResultType = ConvertType(E->getType()); 7030 Value *X = EmitScalarExpr(E->getArg(0)); 7031 Value *Y = EmitScalarExpr(E->getArg(1)); 7032 Value *Z = EmitScalarExpr(E->getArg(2)); 7033 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType); 7034 llvm::Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType); 7035 switch (BuiltinID) { 7036 case PPC::BI__builtin_vsx_xvmaddadp: 7037 case PPC::BI__builtin_vsx_xvmaddasp: 7038 return Builder.CreateCall(F, {X, Y, Z}); 7039 case PPC::BI__builtin_vsx_xvnmaddadp: 7040 case PPC::BI__builtin_vsx_xvnmaddasp: 7041 return Builder.CreateFSub(Zero, 7042 Builder.CreateCall(F, {X, Y, Z}), "sub"); 7043 case PPC::BI__builtin_vsx_xvmsubadp: 7044 case PPC::BI__builtin_vsx_xvmsubasp: 7045 return Builder.CreateCall(F, 7046 {X, Y, Builder.CreateFSub(Zero, Z, "sub")}); 7047 case PPC::BI__builtin_vsx_xvnmsubadp: 7048 case PPC::BI__builtin_vsx_xvnmsubasp: 7049 Value *FsubRes = 7050 Builder.CreateCall(F, {X, Y, Builder.CreateFSub(Zero, Z, "sub")}); 7051 return Builder.CreateFSub(Zero, FsubRes, "sub"); 7052 } 7053 llvm_unreachable("Unknown FMA operation"); 7054 return nullptr; // Suppress no-return warning 7055 } 7056 } 7057 } 7058 7059 Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, 7060 const CallExpr *E) { 7061 switch (BuiltinID) { 7062 case AMDGPU::BI__builtin_amdgcn_div_scale: 7063 case AMDGPU::BI__builtin_amdgcn_div_scalef: { 7064 // Translate from the intrinsics's struct return to the builtin's out 7065 // argument. 7066 7067 Address FlagOutPtr = EmitPointerWithAlignment(E->getArg(3)); 7068 7069 llvm::Value *X = EmitScalarExpr(E->getArg(0)); 7070 llvm::Value *Y = EmitScalarExpr(E->getArg(1)); 7071 llvm::Value *Z = EmitScalarExpr(E->getArg(2)); 7072 7073 llvm::Value *Callee = CGM.getIntrinsic(Intrinsic::amdgcn_div_scale, 7074 X->getType()); 7075 7076 llvm::Value *Tmp = Builder.CreateCall(Callee, {X, Y, Z}); 7077 7078 llvm::Value *Result = Builder.CreateExtractValue(Tmp, 0); 7079 llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1); 7080 7081 llvm::Type *RealFlagType 7082 = FlagOutPtr.getPointer()->getType()->getPointerElementType(); 7083 7084 llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType); 7085 Builder.CreateStore(FlagExt, FlagOutPtr); 7086 return Result; 7087 } 7088 case AMDGPU::BI__builtin_amdgcn_div_fmas: 7089 case AMDGPU::BI__builtin_amdgcn_div_fmasf: { 7090 llvm::Value *Src0 = EmitScalarExpr(E->getArg(0)); 7091 llvm::Value *Src1 = EmitScalarExpr(E->getArg(1)); 7092 llvm::Value *Src2 = EmitScalarExpr(E->getArg(2)); 7093 llvm::Value *Src3 = EmitScalarExpr(E->getArg(3)); 7094 7095 llvm::Value *F = CGM.getIntrinsic(Intrinsic::amdgcn_div_fmas, 7096 Src0->getType()); 7097 llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3); 7098 return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool}); 7099 } 7100 case AMDGPU::BI__builtin_amdgcn_div_fixup: 7101 case AMDGPU::BI__builtin_amdgcn_div_fixupf: 7102 return emitTernaryFPBuiltin(*this, E, Intrinsic::amdgcn_div_fixup); 7103 case AMDGPU::BI__builtin_amdgcn_trig_preop: 7104 case AMDGPU::BI__builtin_amdgcn_trig_preopf: 7105 return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_trig_preop); 7106 case AMDGPU::BI__builtin_amdgcn_rcp: 7107 case AMDGPU::BI__builtin_amdgcn_rcpf: 7108 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rcp); 7109 case AMDGPU::BI__builtin_amdgcn_rsq: 7110 case AMDGPU::BI__builtin_amdgcn_rsqf: 7111 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq); 7112 case AMDGPU::BI__builtin_amdgcn_rsq_clamp: 7113 case AMDGPU::BI__builtin_amdgcn_rsq_clampf: 7114 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq_clamp); 7115 case AMDGPU::BI__builtin_amdgcn_sinf: 7116 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_sin); 7117 case AMDGPU::BI__builtin_amdgcn_cosf: 7118 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_cos); 7119 case AMDGPU::BI__builtin_amdgcn_log_clampf: 7120 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_log_clamp); 7121 case AMDGPU::BI__builtin_amdgcn_ldexp: 7122 case AMDGPU::BI__builtin_amdgcn_ldexpf: 7123 return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_ldexp); 7124 case AMDGPU::BI__builtin_amdgcn_frexp_mant: 7125 case AMDGPU::BI__builtin_amdgcn_frexp_mantf: { 7126 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_frexp_mant); 7127 } 7128 case AMDGPU::BI__builtin_amdgcn_frexp_exp: 7129 case AMDGPU::BI__builtin_amdgcn_frexp_expf: { 7130 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_frexp_exp); 7131 } 7132 case AMDGPU::BI__builtin_amdgcn_fract: 7133 case AMDGPU::BI__builtin_amdgcn_fractf: 7134 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_fract); 7135 case AMDGPU::BI__builtin_amdgcn_class: 7136 case AMDGPU::BI__builtin_amdgcn_classf: 7137 return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_class); 7138 7139 // Legacy amdgpu prefix 7140 case AMDGPU::BI__builtin_amdgpu_rsq: 7141 case AMDGPU::BI__builtin_amdgpu_rsqf: { 7142 if (getTarget().getTriple().getArch() == Triple::amdgcn) 7143 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq); 7144 return emitUnaryBuiltin(*this, E, Intrinsic::r600_rsq); 7145 } 7146 case AMDGPU::BI__builtin_amdgpu_ldexp: 7147 case AMDGPU::BI__builtin_amdgpu_ldexpf: { 7148 if (getTarget().getTriple().getArch() == Triple::amdgcn) 7149 return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_ldexp); 7150 return emitFPIntBuiltin(*this, E, Intrinsic::AMDGPU_ldexp); 7151 } 7152 default: 7153 return nullptr; 7154 } 7155 } 7156 7157 /// Handle a SystemZ function in which the final argument is a pointer 7158 /// to an int that receives the post-instruction CC value. At the LLVM level 7159 /// this is represented as a function that returns a {result, cc} pair. 7160 static Value *EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, 7161 unsigned IntrinsicID, 7162 const CallExpr *E) { 7163 unsigned NumArgs = E->getNumArgs() - 1; 7164 SmallVector<Value *, 8> Args(NumArgs); 7165 for (unsigned I = 0; I < NumArgs; ++I) 7166 Args[I] = CGF.EmitScalarExpr(E->getArg(I)); 7167 Address CCPtr = CGF.EmitPointerWithAlignment(E->getArg(NumArgs)); 7168 Value *F = CGF.CGM.getIntrinsic(IntrinsicID); 7169 Value *Call = CGF.Builder.CreateCall(F, Args); 7170 Value *CC = CGF.Builder.CreateExtractValue(Call, 1); 7171 CGF.Builder.CreateStore(CC, CCPtr); 7172 return CGF.Builder.CreateExtractValue(Call, 0); 7173 } 7174 7175 Value *CodeGenFunction::EmitSystemZBuiltinExpr(unsigned BuiltinID, 7176 const CallExpr *E) { 7177 switch (BuiltinID) { 7178 case SystemZ::BI__builtin_tbegin: { 7179 Value *TDB = EmitScalarExpr(E->getArg(0)); 7180 Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c); 7181 Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin); 7182 return Builder.CreateCall(F, {TDB, Control}); 7183 } 7184 case SystemZ::BI__builtin_tbegin_nofloat: { 7185 Value *TDB = EmitScalarExpr(E->getArg(0)); 7186 Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c); 7187 Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin_nofloat); 7188 return Builder.CreateCall(F, {TDB, Control}); 7189 } 7190 case SystemZ::BI__builtin_tbeginc: { 7191 Value *TDB = llvm::ConstantPointerNull::get(Int8PtrTy); 7192 Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff08); 7193 Value *F = CGM.getIntrinsic(Intrinsic::s390_tbeginc); 7194 return Builder.CreateCall(F, {TDB, Control}); 7195 } 7196 case SystemZ::BI__builtin_tabort: { 7197 Value *Data = EmitScalarExpr(E->getArg(0)); 7198 Value *F = CGM.getIntrinsic(Intrinsic::s390_tabort); 7199 return Builder.CreateCall(F, Builder.CreateSExt(Data, Int64Ty, "tabort")); 7200 } 7201 case SystemZ::BI__builtin_non_tx_store: { 7202 Value *Address = EmitScalarExpr(E->getArg(0)); 7203 Value *Data = EmitScalarExpr(E->getArg(1)); 7204 Value *F = CGM.getIntrinsic(Intrinsic::s390_ntstg); 7205 return Builder.CreateCall(F, {Data, Address}); 7206 } 7207 7208 // Vector builtins. Note that most vector builtins are mapped automatically 7209 // to target-specific LLVM intrinsics. The ones handled specially here can 7210 // be represented via standard LLVM IR, which is preferable to enable common 7211 // LLVM optimizations. 7212 7213 case SystemZ::BI__builtin_s390_vpopctb: 7214 case SystemZ::BI__builtin_s390_vpopcth: 7215 case SystemZ::BI__builtin_s390_vpopctf: 7216 case SystemZ::BI__builtin_s390_vpopctg: { 7217 llvm::Type *ResultType = ConvertType(E->getType()); 7218 Value *X = EmitScalarExpr(E->getArg(0)); 7219 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType); 7220 return Builder.CreateCall(F, X); 7221 } 7222 7223 case SystemZ::BI__builtin_s390_vclzb: 7224 case SystemZ::BI__builtin_s390_vclzh: 7225 case SystemZ::BI__builtin_s390_vclzf: 7226 case SystemZ::BI__builtin_s390_vclzg: { 7227 llvm::Type *ResultType = ConvertType(E->getType()); 7228 Value *X = EmitScalarExpr(E->getArg(0)); 7229 Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false); 7230 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType); 7231 return Builder.CreateCall(F, {X, Undef}); 7232 } 7233 7234 case SystemZ::BI__builtin_s390_vctzb: 7235 case SystemZ::BI__builtin_s390_vctzh: 7236 case SystemZ::BI__builtin_s390_vctzf: 7237 case SystemZ::BI__builtin_s390_vctzg: { 7238 llvm::Type *ResultType = ConvertType(E->getType()); 7239 Value *X = EmitScalarExpr(E->getArg(0)); 7240 Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false); 7241 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType); 7242 return Builder.CreateCall(F, {X, Undef}); 7243 } 7244 7245 case SystemZ::BI__builtin_s390_vfsqdb: { 7246 llvm::Type *ResultType = ConvertType(E->getType()); 7247 Value *X = EmitScalarExpr(E->getArg(0)); 7248 Function *F = CGM.getIntrinsic(Intrinsic::sqrt, ResultType); 7249 return Builder.CreateCall(F, X); 7250 } 7251 case SystemZ::BI__builtin_s390_vfmadb: { 7252 llvm::Type *ResultType = ConvertType(E->getType()); 7253 Value *X = EmitScalarExpr(E->getArg(0)); 7254 Value *Y = EmitScalarExpr(E->getArg(1)); 7255 Value *Z = EmitScalarExpr(E->getArg(2)); 7256 Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType); 7257 return Builder.CreateCall(F, {X, Y, Z}); 7258 } 7259 case SystemZ::BI__builtin_s390_vfmsdb: { 7260 llvm::Type *ResultType = ConvertType(E->getType()); 7261 Value *X = EmitScalarExpr(E->getArg(0)); 7262 Value *Y = EmitScalarExpr(E->getArg(1)); 7263 Value *Z = EmitScalarExpr(E->getArg(2)); 7264 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType); 7265 Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType); 7266 return Builder.CreateCall(F, {X, Y, Builder.CreateFSub(Zero, Z, "sub")}); 7267 } 7268 case SystemZ::BI__builtin_s390_vflpdb: { 7269 llvm::Type *ResultType = ConvertType(E->getType()); 7270 Value *X = EmitScalarExpr(E->getArg(0)); 7271 Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType); 7272 return Builder.CreateCall(F, X); 7273 } 7274 case SystemZ::BI__builtin_s390_vflndb: { 7275 llvm::Type *ResultType = ConvertType(E->getType()); 7276 Value *X = EmitScalarExpr(E->getArg(0)); 7277 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType); 7278 Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType); 7279 return Builder.CreateFSub(Zero, Builder.CreateCall(F, X), "sub"); 7280 } 7281 case SystemZ::BI__builtin_s390_vfidb: { 7282 llvm::Type *ResultType = ConvertType(E->getType()); 7283 Value *X = EmitScalarExpr(E->getArg(0)); 7284 // Constant-fold the M4 and M5 mask arguments. 7285 llvm::APSInt M4, M5; 7286 bool IsConstM4 = E->getArg(1)->isIntegerConstantExpr(M4, getContext()); 7287 bool IsConstM5 = E->getArg(2)->isIntegerConstantExpr(M5, getContext()); 7288 assert(IsConstM4 && IsConstM5 && "Constant arg isn't actually constant?"); 7289 (void)IsConstM4; (void)IsConstM5; 7290 // Check whether this instance of vfidb can be represented via a LLVM 7291 // standard intrinsic. We only support some combinations of M4 and M5. 7292 Intrinsic::ID ID = Intrinsic::not_intrinsic; 7293 switch (M4.getZExtValue()) { 7294 default: break; 7295 case 0: // IEEE-inexact exception allowed 7296 switch (M5.getZExtValue()) { 7297 default: break; 7298 case 0: ID = Intrinsic::rint; break; 7299 } 7300 break; 7301 case 4: // IEEE-inexact exception suppressed 7302 switch (M5.getZExtValue()) { 7303 default: break; 7304 case 0: ID = Intrinsic::nearbyint; break; 7305 case 1: ID = Intrinsic::round; break; 7306 case 5: ID = Intrinsic::trunc; break; 7307 case 6: ID = Intrinsic::ceil; break; 7308 case 7: ID = Intrinsic::floor; break; 7309 } 7310 break; 7311 } 7312 if (ID != Intrinsic::not_intrinsic) { 7313 Function *F = CGM.getIntrinsic(ID, ResultType); 7314 return Builder.CreateCall(F, X); 7315 } 7316 Function *F = CGM.getIntrinsic(Intrinsic::s390_vfidb); 7317 Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4); 7318 Value *M5Value = llvm::ConstantInt::get(getLLVMContext(), M5); 7319 return Builder.CreateCall(F, {X, M4Value, M5Value}); 7320 } 7321 7322 // Vector intrisincs that output the post-instruction CC value. 7323 7324 #define INTRINSIC_WITH_CC(NAME) \ 7325 case SystemZ::BI__builtin_##NAME: \ 7326 return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E) 7327 7328 INTRINSIC_WITH_CC(s390_vpkshs); 7329 INTRINSIC_WITH_CC(s390_vpksfs); 7330 INTRINSIC_WITH_CC(s390_vpksgs); 7331 7332 INTRINSIC_WITH_CC(s390_vpklshs); 7333 INTRINSIC_WITH_CC(s390_vpklsfs); 7334 INTRINSIC_WITH_CC(s390_vpklsgs); 7335 7336 INTRINSIC_WITH_CC(s390_vceqbs); 7337 INTRINSIC_WITH_CC(s390_vceqhs); 7338 INTRINSIC_WITH_CC(s390_vceqfs); 7339 INTRINSIC_WITH_CC(s390_vceqgs); 7340 7341 INTRINSIC_WITH_CC(s390_vchbs); 7342 INTRINSIC_WITH_CC(s390_vchhs); 7343 INTRINSIC_WITH_CC(s390_vchfs); 7344 INTRINSIC_WITH_CC(s390_vchgs); 7345 7346 INTRINSIC_WITH_CC(s390_vchlbs); 7347 INTRINSIC_WITH_CC(s390_vchlhs); 7348 INTRINSIC_WITH_CC(s390_vchlfs); 7349 INTRINSIC_WITH_CC(s390_vchlgs); 7350 7351 INTRINSIC_WITH_CC(s390_vfaebs); 7352 INTRINSIC_WITH_CC(s390_vfaehs); 7353 INTRINSIC_WITH_CC(s390_vfaefs); 7354 7355 INTRINSIC_WITH_CC(s390_vfaezbs); 7356 INTRINSIC_WITH_CC(s390_vfaezhs); 7357 INTRINSIC_WITH_CC(s390_vfaezfs); 7358 7359 INTRINSIC_WITH_CC(s390_vfeebs); 7360 INTRINSIC_WITH_CC(s390_vfeehs); 7361 INTRINSIC_WITH_CC(s390_vfeefs); 7362 7363 INTRINSIC_WITH_CC(s390_vfeezbs); 7364 INTRINSIC_WITH_CC(s390_vfeezhs); 7365 INTRINSIC_WITH_CC(s390_vfeezfs); 7366 7367 INTRINSIC_WITH_CC(s390_vfenebs); 7368 INTRINSIC_WITH_CC(s390_vfenehs); 7369 INTRINSIC_WITH_CC(s390_vfenefs); 7370 7371 INTRINSIC_WITH_CC(s390_vfenezbs); 7372 INTRINSIC_WITH_CC(s390_vfenezhs); 7373 INTRINSIC_WITH_CC(s390_vfenezfs); 7374 7375 INTRINSIC_WITH_CC(s390_vistrbs); 7376 INTRINSIC_WITH_CC(s390_vistrhs); 7377 INTRINSIC_WITH_CC(s390_vistrfs); 7378 7379 INTRINSIC_WITH_CC(s390_vstrcbs); 7380 INTRINSIC_WITH_CC(s390_vstrchs); 7381 INTRINSIC_WITH_CC(s390_vstrcfs); 7382 7383 INTRINSIC_WITH_CC(s390_vstrczbs); 7384 INTRINSIC_WITH_CC(s390_vstrczhs); 7385 INTRINSIC_WITH_CC(s390_vstrczfs); 7386 7387 INTRINSIC_WITH_CC(s390_vfcedbs); 7388 INTRINSIC_WITH_CC(s390_vfchdbs); 7389 INTRINSIC_WITH_CC(s390_vfchedbs); 7390 7391 INTRINSIC_WITH_CC(s390_vftcidb); 7392 7393 #undef INTRINSIC_WITH_CC 7394 7395 default: 7396 return nullptr; 7397 } 7398 } 7399 7400 Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, 7401 const CallExpr *E) { 7402 auto MakeLdg = [&](unsigned IntrinsicID) { 7403 Value *Ptr = EmitScalarExpr(E->getArg(0)); 7404 AlignmentSource AlignSource; 7405 clang::CharUnits Align = 7406 getNaturalPointeeTypeAlignment(E->getArg(0)->getType(), &AlignSource); 7407 return Builder.CreateCall( 7408 CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(), 7409 Ptr->getType()}), 7410 {Ptr, ConstantInt::get(Builder.getInt32Ty(), Align.getQuantity())}); 7411 }; 7412 7413 switch (BuiltinID) { 7414 case NVPTX::BI__nvvm_atom_add_gen_i: 7415 case NVPTX::BI__nvvm_atom_add_gen_l: 7416 case NVPTX::BI__nvvm_atom_add_gen_ll: 7417 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Add, E); 7418 7419 case NVPTX::BI__nvvm_atom_sub_gen_i: 7420 case NVPTX::BI__nvvm_atom_sub_gen_l: 7421 case NVPTX::BI__nvvm_atom_sub_gen_ll: 7422 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Sub, E); 7423 7424 case NVPTX::BI__nvvm_atom_and_gen_i: 7425 case NVPTX::BI__nvvm_atom_and_gen_l: 7426 case NVPTX::BI__nvvm_atom_and_gen_ll: 7427 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::And, E); 7428 7429 case NVPTX::BI__nvvm_atom_or_gen_i: 7430 case NVPTX::BI__nvvm_atom_or_gen_l: 7431 case NVPTX::BI__nvvm_atom_or_gen_ll: 7432 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Or, E); 7433 7434 case NVPTX::BI__nvvm_atom_xor_gen_i: 7435 case NVPTX::BI__nvvm_atom_xor_gen_l: 7436 case NVPTX::BI__nvvm_atom_xor_gen_ll: 7437 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xor, E); 7438 7439 case NVPTX::BI__nvvm_atom_xchg_gen_i: 7440 case NVPTX::BI__nvvm_atom_xchg_gen_l: 7441 case NVPTX::BI__nvvm_atom_xchg_gen_ll: 7442 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xchg, E); 7443 7444 case NVPTX::BI__nvvm_atom_max_gen_i: 7445 case NVPTX::BI__nvvm_atom_max_gen_l: 7446 case NVPTX::BI__nvvm_atom_max_gen_ll: 7447 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Max, E); 7448 7449 case NVPTX::BI__nvvm_atom_max_gen_ui: 7450 case NVPTX::BI__nvvm_atom_max_gen_ul: 7451 case NVPTX::BI__nvvm_atom_max_gen_ull: 7452 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMax, E); 7453 7454 case NVPTX::BI__nvvm_atom_min_gen_i: 7455 case NVPTX::BI__nvvm_atom_min_gen_l: 7456 case NVPTX::BI__nvvm_atom_min_gen_ll: 7457 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Min, E); 7458 7459 case NVPTX::BI__nvvm_atom_min_gen_ui: 7460 case NVPTX::BI__nvvm_atom_min_gen_ul: 7461 case NVPTX::BI__nvvm_atom_min_gen_ull: 7462 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMin, E); 7463 7464 case NVPTX::BI__nvvm_atom_cas_gen_i: 7465 case NVPTX::BI__nvvm_atom_cas_gen_l: 7466 case NVPTX::BI__nvvm_atom_cas_gen_ll: 7467 // __nvvm_atom_cas_gen_* should return the old value rather than the 7468 // success flag. 7469 return MakeAtomicCmpXchgValue(*this, E, /*ReturnBool=*/false); 7470 7471 case NVPTX::BI__nvvm_atom_add_gen_f: { 7472 Value *Ptr = EmitScalarExpr(E->getArg(0)); 7473 Value *Val = EmitScalarExpr(E->getArg(1)); 7474 // atomicrmw only deals with integer arguments so we need to use 7475 // LLVM's nvvm_atomic_load_add_f32 intrinsic for that. 7476 Value *FnALAF32 = 7477 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_add_f32, Ptr->getType()); 7478 return Builder.CreateCall(FnALAF32, {Ptr, Val}); 7479 } 7480 7481 case NVPTX::BI__nvvm_atom_inc_gen_ui: { 7482 Value *Ptr = EmitScalarExpr(E->getArg(0)); 7483 Value *Val = EmitScalarExpr(E->getArg(1)); 7484 Value *FnALI32 = 7485 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_inc_32, Ptr->getType()); 7486 return Builder.CreateCall(FnALI32, {Ptr, Val}); 7487 } 7488 7489 case NVPTX::BI__nvvm_atom_dec_gen_ui: { 7490 Value *Ptr = EmitScalarExpr(E->getArg(0)); 7491 Value *Val = EmitScalarExpr(E->getArg(1)); 7492 Value *FnALD32 = 7493 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_dec_32, Ptr->getType()); 7494 return Builder.CreateCall(FnALD32, {Ptr, Val}); 7495 } 7496 7497 case NVPTX::BI__nvvm_ldg_c: 7498 case NVPTX::BI__nvvm_ldg_c2: 7499 case NVPTX::BI__nvvm_ldg_c4: 7500 case NVPTX::BI__nvvm_ldg_s: 7501 case NVPTX::BI__nvvm_ldg_s2: 7502 case NVPTX::BI__nvvm_ldg_s4: 7503 case NVPTX::BI__nvvm_ldg_i: 7504 case NVPTX::BI__nvvm_ldg_i2: 7505 case NVPTX::BI__nvvm_ldg_i4: 7506 case NVPTX::BI__nvvm_ldg_l: 7507 case NVPTX::BI__nvvm_ldg_ll: 7508 case NVPTX::BI__nvvm_ldg_ll2: 7509 case NVPTX::BI__nvvm_ldg_uc: 7510 case NVPTX::BI__nvvm_ldg_uc2: 7511 case NVPTX::BI__nvvm_ldg_uc4: 7512 case NVPTX::BI__nvvm_ldg_us: 7513 case NVPTX::BI__nvvm_ldg_us2: 7514 case NVPTX::BI__nvvm_ldg_us4: 7515 case NVPTX::BI__nvvm_ldg_ui: 7516 case NVPTX::BI__nvvm_ldg_ui2: 7517 case NVPTX::BI__nvvm_ldg_ui4: 7518 case NVPTX::BI__nvvm_ldg_ul: 7519 case NVPTX::BI__nvvm_ldg_ull: 7520 case NVPTX::BI__nvvm_ldg_ull2: 7521 // PTX Interoperability section 2.2: "For a vector with an even number of 7522 // elements, its alignment is set to number of elements times the alignment 7523 // of its member: n*alignof(t)." 7524 return MakeLdg(Intrinsic::nvvm_ldg_global_i); 7525 case NVPTX::BI__nvvm_ldg_f: 7526 case NVPTX::BI__nvvm_ldg_f2: 7527 case NVPTX::BI__nvvm_ldg_f4: 7528 case NVPTX::BI__nvvm_ldg_d: 7529 case NVPTX::BI__nvvm_ldg_d2: 7530 return MakeLdg(Intrinsic::nvvm_ldg_global_f); 7531 default: 7532 return nullptr; 7533 } 7534 } 7535 7536 Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, 7537 const CallExpr *E) { 7538 switch (BuiltinID) { 7539 case WebAssembly::BI__builtin_wasm_current_memory: { 7540 llvm::Type *ResultType = ConvertType(E->getType()); 7541 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_current_memory, ResultType); 7542 return Builder.CreateCall(Callee); 7543 } 7544 case WebAssembly::BI__builtin_wasm_grow_memory: { 7545 Value *X = EmitScalarExpr(E->getArg(0)); 7546 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_grow_memory, X->getType()); 7547 return Builder.CreateCall(Callee, X); 7548 } 7549 7550 default: 7551 return nullptr; 7552 } 7553 } 7554