1 //===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This contains code to emit Builtin calls as LLVM code. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "CodeGenFunction.h" 15 #include "CGCXXABI.h" 16 #include "CGObjCRuntime.h" 17 #include "CodeGenModule.h" 18 #include "TargetInfo.h" 19 #include "clang/AST/ASTContext.h" 20 #include "clang/AST/Decl.h" 21 #include "clang/Basic/TargetBuiltins.h" 22 #include "clang/Basic/TargetInfo.h" 23 #include "clang/CodeGen/CGFunctionInfo.h" 24 #include "llvm/ADT/StringExtras.h" 25 #include "llvm/IR/CallSite.h" 26 #include "llvm/IR/DataLayout.h" 27 #include "llvm/IR/InlineAsm.h" 28 #include "llvm/IR/Intrinsics.h" 29 #include <sstream> 30 31 using namespace clang; 32 using namespace CodeGen; 33 using namespace llvm; 34 35 /// getBuiltinLibFunction - Given a builtin id for a function like 36 /// "__builtin_fabsf", return a Function* for "fabsf". 37 llvm::Value *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD, 38 unsigned BuiltinID) { 39 assert(Context.BuiltinInfo.isLibFunction(BuiltinID)); 40 41 // Get the name, skip over the __builtin_ prefix (if necessary). 42 StringRef Name; 43 GlobalDecl D(FD); 44 45 // If the builtin has been declared explicitly with an assembler label, 46 // use the mangled name. This differs from the plain label on platforms 47 // that prefix labels. 48 if (FD->hasAttr<AsmLabelAttr>()) 49 Name = getMangledName(D); 50 else 51 Name = Context.BuiltinInfo.getName(BuiltinID) + 10; 52 53 llvm::FunctionType *Ty = 54 cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType())); 55 56 return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false); 57 } 58 59 /// Emit the conversions required to turn the given value into an 60 /// integer of the given size. 61 static Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V, 62 QualType T, llvm::IntegerType *IntType) { 63 V = CGF.EmitToMemory(V, T); 64 65 if (V->getType()->isPointerTy()) 66 return CGF.Builder.CreatePtrToInt(V, IntType); 67 68 assert(V->getType() == IntType); 69 return V; 70 } 71 72 static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, 73 QualType T, llvm::Type *ResultType) { 74 V = CGF.EmitFromMemory(V, T); 75 76 if (ResultType->isPointerTy()) 77 return CGF.Builder.CreateIntToPtr(V, ResultType); 78 79 assert(V->getType() == ResultType); 80 return V; 81 } 82 83 /// Utility to insert an atomic instruction based on Instrinsic::ID 84 /// and the expression node. 85 static Value *MakeBinaryAtomicValue(CodeGenFunction &CGF, 86 llvm::AtomicRMWInst::BinOp Kind, 87 const CallExpr *E) { 88 QualType T = E->getType(); 89 assert(E->getArg(0)->getType()->isPointerType()); 90 assert(CGF.getContext().hasSameUnqualifiedType(T, 91 E->getArg(0)->getType()->getPointeeType())); 92 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType())); 93 94 llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); 95 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 96 97 llvm::IntegerType *IntType = 98 llvm::IntegerType::get(CGF.getLLVMContext(), 99 CGF.getContext().getTypeSize(T)); 100 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 101 102 llvm::Value *Args[2]; 103 Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); 104 Args[1] = CGF.EmitScalarExpr(E->getArg(1)); 105 llvm::Type *ValueType = Args[1]->getType(); 106 Args[1] = EmitToInt(CGF, Args[1], T, IntType); 107 108 llvm::Value *Result = CGF.Builder.CreateAtomicRMW( 109 Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent); 110 return EmitFromInt(CGF, Result, T, ValueType); 111 } 112 113 static Value *EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) { 114 Value *Val = CGF.EmitScalarExpr(E->getArg(0)); 115 Value *Address = CGF.EmitScalarExpr(E->getArg(1)); 116 117 // Convert the type of the pointer to a pointer to the stored type. 118 Val = CGF.EmitToMemory(Val, E->getArg(0)->getType()); 119 Value *BC = CGF.Builder.CreateBitCast( 120 Address, llvm::PointerType::getUnqual(Val->getType()), "cast"); 121 LValue LV = CGF.MakeNaturalAlignAddrLValue(BC, E->getArg(0)->getType()); 122 LV.setNontemporal(true); 123 CGF.EmitStoreOfScalar(Val, LV, false); 124 return nullptr; 125 } 126 127 static Value *EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) { 128 Value *Address = CGF.EmitScalarExpr(E->getArg(0)); 129 130 LValue LV = CGF.MakeNaturalAlignAddrLValue(Address, E->getType()); 131 LV.setNontemporal(true); 132 return CGF.EmitLoadOfScalar(LV, E->getExprLoc()); 133 } 134 135 static RValue EmitBinaryAtomic(CodeGenFunction &CGF, 136 llvm::AtomicRMWInst::BinOp Kind, 137 const CallExpr *E) { 138 return RValue::get(MakeBinaryAtomicValue(CGF, Kind, E)); 139 } 140 141 /// Utility to insert an atomic instruction based Instrinsic::ID and 142 /// the expression node, where the return value is the result of the 143 /// operation. 144 static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, 145 llvm::AtomicRMWInst::BinOp Kind, 146 const CallExpr *E, 147 Instruction::BinaryOps Op, 148 bool Invert = false) { 149 QualType T = E->getType(); 150 assert(E->getArg(0)->getType()->isPointerType()); 151 assert(CGF.getContext().hasSameUnqualifiedType(T, 152 E->getArg(0)->getType()->getPointeeType())); 153 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType())); 154 155 llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); 156 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 157 158 llvm::IntegerType *IntType = 159 llvm::IntegerType::get(CGF.getLLVMContext(), 160 CGF.getContext().getTypeSize(T)); 161 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 162 163 llvm::Value *Args[2]; 164 Args[1] = CGF.EmitScalarExpr(E->getArg(1)); 165 llvm::Type *ValueType = Args[1]->getType(); 166 Args[1] = EmitToInt(CGF, Args[1], T, IntType); 167 Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); 168 169 llvm::Value *Result = CGF.Builder.CreateAtomicRMW( 170 Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent); 171 Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]); 172 if (Invert) 173 Result = CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result, 174 llvm::ConstantInt::get(IntType, -1)); 175 Result = EmitFromInt(CGF, Result, T, ValueType); 176 return RValue::get(Result); 177 } 178 179 /// @brief Utility to insert an atomic cmpxchg instruction. 180 /// 181 /// @param CGF The current codegen function. 182 /// @param E Builtin call expression to convert to cmpxchg. 183 /// arg0 - address to operate on 184 /// arg1 - value to compare with 185 /// arg2 - new value 186 /// @param ReturnBool Specifies whether to return success flag of 187 /// cmpxchg result or the old value. 188 /// 189 /// @returns result of cmpxchg, according to ReturnBool 190 static Value *MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, 191 bool ReturnBool) { 192 QualType T = ReturnBool ? E->getArg(1)->getType() : E->getType(); 193 llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); 194 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 195 196 llvm::IntegerType *IntType = llvm::IntegerType::get( 197 CGF.getLLVMContext(), CGF.getContext().getTypeSize(T)); 198 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 199 200 Value *Args[3]; 201 Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); 202 Args[1] = CGF.EmitScalarExpr(E->getArg(1)); 203 llvm::Type *ValueType = Args[1]->getType(); 204 Args[1] = EmitToInt(CGF, Args[1], T, IntType); 205 Args[2] = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType); 206 207 Value *Pair = CGF.Builder.CreateAtomicCmpXchg( 208 Args[0], Args[1], Args[2], llvm::AtomicOrdering::SequentiallyConsistent, 209 llvm::AtomicOrdering::SequentiallyConsistent); 210 if (ReturnBool) 211 // Extract boolean success flag and zext it to int. 212 return CGF.Builder.CreateZExt(CGF.Builder.CreateExtractValue(Pair, 1), 213 CGF.ConvertType(E->getType())); 214 else 215 // Extract old value and emit it using the same type as compare value. 216 return EmitFromInt(CGF, CGF.Builder.CreateExtractValue(Pair, 0), T, 217 ValueType); 218 } 219 220 /// EmitFAbs - Emit a call to @llvm.fabs(). 221 static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) { 222 Value *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType()); 223 llvm::CallInst *Call = CGF.Builder.CreateCall(F, V); 224 Call->setDoesNotAccessMemory(); 225 return Call; 226 } 227 228 /// Emit the computation of the sign bit for a floating point value. Returns 229 /// the i1 sign bit value. 230 static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) { 231 LLVMContext &C = CGF.CGM.getLLVMContext(); 232 233 llvm::Type *Ty = V->getType(); 234 int Width = Ty->getPrimitiveSizeInBits(); 235 llvm::Type *IntTy = llvm::IntegerType::get(C, Width); 236 V = CGF.Builder.CreateBitCast(V, IntTy); 237 if (Ty->isPPC_FP128Ty()) { 238 // We want the sign bit of the higher-order double. The bitcast we just 239 // did works as if the double-double was stored to memory and then 240 // read as an i128. The "store" will put the higher-order double in the 241 // lower address in both little- and big-Endian modes, but the "load" 242 // will treat those bits as a different part of the i128: the low bits in 243 // little-Endian, the high bits in big-Endian. Therefore, on big-Endian 244 // we need to shift the high bits down to the low before truncating. 245 Width >>= 1; 246 if (CGF.getTarget().isBigEndian()) { 247 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width); 248 V = CGF.Builder.CreateLShr(V, ShiftCst); 249 } 250 // We are truncating value in order to extract the higher-order 251 // double, which we will be using to extract the sign from. 252 IntTy = llvm::IntegerType::get(C, Width); 253 V = CGF.Builder.CreateTrunc(V, IntTy); 254 } 255 Value *Zero = llvm::Constant::getNullValue(IntTy); 256 return CGF.Builder.CreateICmpSLT(V, Zero); 257 } 258 259 static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *Fn, 260 const CallExpr *E, llvm::Value *calleeValue) { 261 return CGF.EmitCall(E->getCallee()->getType(), calleeValue, E, 262 ReturnValueSlot(), Fn); 263 } 264 265 /// \brief Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.* 266 /// depending on IntrinsicID. 267 /// 268 /// \arg CGF The current codegen function. 269 /// \arg IntrinsicID The ID for the Intrinsic we wish to generate. 270 /// \arg X The first argument to the llvm.*.with.overflow.*. 271 /// \arg Y The second argument to the llvm.*.with.overflow.*. 272 /// \arg Carry The carry returned by the llvm.*.with.overflow.*. 273 /// \returns The result (i.e. sum/product) returned by the intrinsic. 274 static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF, 275 const llvm::Intrinsic::ID IntrinsicID, 276 llvm::Value *X, llvm::Value *Y, 277 llvm::Value *&Carry) { 278 // Make sure we have integers of the same width. 279 assert(X->getType() == Y->getType() && 280 "Arguments must be the same type. (Did you forget to make sure both " 281 "arguments have the same integer width?)"); 282 283 llvm::Value *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType()); 284 llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, {X, Y}); 285 Carry = CGF.Builder.CreateExtractValue(Tmp, 1); 286 return CGF.Builder.CreateExtractValue(Tmp, 0); 287 } 288 289 // Emit a simple mangled intrinsic that has 1 argument and a return type 290 // matching the argument type. 291 static Value *emitUnaryBuiltin(CodeGenFunction &CGF, 292 const CallExpr *E, 293 unsigned IntrinsicID) { 294 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); 295 296 Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); 297 return CGF.Builder.CreateCall(F, Src0); 298 } 299 300 // Emit an intrinsic that has 3 float or double operands. 301 static Value *emitTernaryFPBuiltin(CodeGenFunction &CGF, 302 const CallExpr *E, 303 unsigned IntrinsicID) { 304 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); 305 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); 306 llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2)); 307 308 Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); 309 return CGF.Builder.CreateCall(F, {Src0, Src1, Src2}); 310 } 311 312 // Emit an intrinsic that has 1 float or double operand, and 1 integer. 313 static Value *emitFPIntBuiltin(CodeGenFunction &CGF, 314 const CallExpr *E, 315 unsigned IntrinsicID) { 316 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); 317 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); 318 319 Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); 320 return CGF.Builder.CreateCall(F, {Src0, Src1}); 321 } 322 323 namespace { 324 struct WidthAndSignedness { 325 unsigned Width; 326 bool Signed; 327 }; 328 } 329 330 static WidthAndSignedness 331 getIntegerWidthAndSignedness(const clang::ASTContext &context, 332 const clang::QualType Type) { 333 assert(Type->isIntegerType() && "Given type is not an integer."); 334 unsigned Width = Type->isBooleanType() ? 1 : context.getTypeInfo(Type).Width; 335 bool Signed = Type->isSignedIntegerType(); 336 return {Width, Signed}; 337 } 338 339 // Given one or more integer types, this function produces an integer type that 340 // encompasses them: any value in one of the given types could be expressed in 341 // the encompassing type. 342 static struct WidthAndSignedness 343 EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) { 344 assert(Types.size() > 0 && "Empty list of types."); 345 346 // If any of the given types is signed, we must return a signed type. 347 bool Signed = false; 348 for (const auto &Type : Types) { 349 Signed |= Type.Signed; 350 } 351 352 // The encompassing type must have a width greater than or equal to the width 353 // of the specified types. Aditionally, if the encompassing type is signed, 354 // its width must be strictly greater than the width of any unsigned types 355 // given. 356 unsigned Width = 0; 357 for (const auto &Type : Types) { 358 unsigned MinWidth = Type.Width + (Signed && !Type.Signed); 359 if (Width < MinWidth) { 360 Width = MinWidth; 361 } 362 } 363 364 return {Width, Signed}; 365 } 366 367 Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) { 368 llvm::Type *DestType = Int8PtrTy; 369 if (ArgValue->getType() != DestType) 370 ArgValue = 371 Builder.CreateBitCast(ArgValue, DestType, ArgValue->getName().data()); 372 373 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend; 374 return Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue); 375 } 376 377 /// Checks if using the result of __builtin_object_size(p, @p From) in place of 378 /// __builtin_object_size(p, @p To) is correct 379 static bool areBOSTypesCompatible(int From, int To) { 380 // Note: Our __builtin_object_size implementation currently treats Type=0 and 381 // Type=2 identically. Encoding this implementation detail here may make 382 // improving __builtin_object_size difficult in the future, so it's omitted. 383 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2); 384 } 385 386 static llvm::Value * 387 getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) { 388 return ConstantInt::get(ResType, (Type & 2) ? 0 : -1, /*isSigned=*/true); 389 } 390 391 llvm::Value * 392 CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type, 393 llvm::IntegerType *ResType) { 394 uint64_t ObjectSize; 395 if (!E->tryEvaluateObjectSize(ObjectSize, getContext(), Type)) 396 return emitBuiltinObjectSize(E, Type, ResType); 397 return ConstantInt::get(ResType, ObjectSize, /*isSigned=*/true); 398 } 399 400 /// Returns a Value corresponding to the size of the given expression. 401 /// This Value may be either of the following: 402 /// - A llvm::Argument (if E is a param with the pass_object_size attribute on 403 /// it) 404 /// - A call to the @llvm.objectsize intrinsic 405 llvm::Value * 406 CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type, 407 llvm::IntegerType *ResType) { 408 // We need to reference an argument if the pointer is a parameter with the 409 // pass_object_size attribute. 410 if (auto *D = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts())) { 411 auto *Param = dyn_cast<ParmVarDecl>(D->getDecl()); 412 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>(); 413 if (Param != nullptr && PS != nullptr && 414 areBOSTypesCompatible(PS->getType(), Type)) { 415 auto Iter = SizeArguments.find(Param); 416 assert(Iter != SizeArguments.end()); 417 418 const ImplicitParamDecl *D = Iter->second; 419 auto DIter = LocalDeclMap.find(D); 420 assert(DIter != LocalDeclMap.end()); 421 422 return EmitLoadOfScalar(DIter->second, /*volatile=*/false, 423 getContext().getSizeType(), E->getLocStart()); 424 } 425 } 426 427 // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't 428 // evaluate E for side-effects. In either case, we shouldn't lower to 429 // @llvm.objectsize. 430 if (Type == 3 || E->HasSideEffects(getContext())) 431 return getDefaultBuiltinObjectSizeResult(Type, ResType); 432 433 // LLVM only supports 0 and 2, make sure that we pass along that 434 // as a boolean. 435 auto *CI = ConstantInt::get(Builder.getInt1Ty(), (Type & 2) >> 1); 436 // FIXME: Get right address space. 437 llvm::Type *Tys[] = {ResType, Builder.getInt8PtrTy(0)}; 438 Value *F = CGM.getIntrinsic(Intrinsic::objectsize, Tys); 439 return Builder.CreateCall(F, {EmitScalarExpr(E), CI}); 440 } 441 442 RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, 443 unsigned BuiltinID, const CallExpr *E, 444 ReturnValueSlot ReturnValue) { 445 // See if we can constant fold this builtin. If so, don't emit it at all. 446 Expr::EvalResult Result; 447 if (E->EvaluateAsRValue(Result, CGM.getContext()) && 448 !Result.hasSideEffects()) { 449 if (Result.Val.isInt()) 450 return RValue::get(llvm::ConstantInt::get(getLLVMContext(), 451 Result.Val.getInt())); 452 if (Result.Val.isFloat()) 453 return RValue::get(llvm::ConstantFP::get(getLLVMContext(), 454 Result.Val.getFloat())); 455 } 456 457 switch (BuiltinID) { 458 default: break; // Handle intrinsics and libm functions below. 459 case Builtin::BI__builtin___CFStringMakeConstantString: 460 case Builtin::BI__builtin___NSStringMakeConstantString: 461 return RValue::get(CGM.EmitConstantExpr(E, E->getType(), nullptr)); 462 case Builtin::BI__builtin_stdarg_start: 463 case Builtin::BI__builtin_va_start: 464 case Builtin::BI__va_start: 465 case Builtin::BI__builtin_va_end: 466 return RValue::get( 467 EmitVAStartEnd(BuiltinID == Builtin::BI__va_start 468 ? EmitScalarExpr(E->getArg(0)) 469 : EmitVAListRef(E->getArg(0)).getPointer(), 470 BuiltinID != Builtin::BI__builtin_va_end)); 471 case Builtin::BI__builtin_va_copy: { 472 Value *DstPtr = EmitVAListRef(E->getArg(0)).getPointer(); 473 Value *SrcPtr = EmitVAListRef(E->getArg(1)).getPointer(); 474 475 llvm::Type *Type = Int8PtrTy; 476 477 DstPtr = Builder.CreateBitCast(DstPtr, Type); 478 SrcPtr = Builder.CreateBitCast(SrcPtr, Type); 479 return RValue::get(Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy), 480 {DstPtr, SrcPtr})); 481 } 482 case Builtin::BI__builtin_abs: 483 case Builtin::BI__builtin_labs: 484 case Builtin::BI__builtin_llabs: { 485 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 486 487 Value *NegOp = Builder.CreateNeg(ArgValue, "neg"); 488 Value *CmpResult = 489 Builder.CreateICmpSGE(ArgValue, 490 llvm::Constant::getNullValue(ArgValue->getType()), 491 "abscond"); 492 Value *Result = 493 Builder.CreateSelect(CmpResult, ArgValue, NegOp, "abs"); 494 495 return RValue::get(Result); 496 } 497 case Builtin::BI__builtin_fabs: 498 case Builtin::BI__builtin_fabsf: 499 case Builtin::BI__builtin_fabsl: { 500 Value *Arg1 = EmitScalarExpr(E->getArg(0)); 501 Value *Result = EmitFAbs(*this, Arg1); 502 return RValue::get(Result); 503 } 504 case Builtin::BI__builtin_fmod: 505 case Builtin::BI__builtin_fmodf: 506 case Builtin::BI__builtin_fmodl: { 507 Value *Arg1 = EmitScalarExpr(E->getArg(0)); 508 Value *Arg2 = EmitScalarExpr(E->getArg(1)); 509 Value *Result = Builder.CreateFRem(Arg1, Arg2, "fmod"); 510 return RValue::get(Result); 511 } 512 513 case Builtin::BI__builtin_conj: 514 case Builtin::BI__builtin_conjf: 515 case Builtin::BI__builtin_conjl: { 516 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 517 Value *Real = ComplexVal.first; 518 Value *Imag = ComplexVal.second; 519 Value *Zero = 520 Imag->getType()->isFPOrFPVectorTy() 521 ? llvm::ConstantFP::getZeroValueForNegation(Imag->getType()) 522 : llvm::Constant::getNullValue(Imag->getType()); 523 524 Imag = Builder.CreateFSub(Zero, Imag, "sub"); 525 return RValue::getComplex(std::make_pair(Real, Imag)); 526 } 527 case Builtin::BI__builtin_creal: 528 case Builtin::BI__builtin_crealf: 529 case Builtin::BI__builtin_creall: 530 case Builtin::BIcreal: 531 case Builtin::BIcrealf: 532 case Builtin::BIcreall: { 533 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 534 return RValue::get(ComplexVal.first); 535 } 536 537 case Builtin::BI__builtin_cimag: 538 case Builtin::BI__builtin_cimagf: 539 case Builtin::BI__builtin_cimagl: 540 case Builtin::BIcimag: 541 case Builtin::BIcimagf: 542 case Builtin::BIcimagl: { 543 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 544 return RValue::get(ComplexVal.second); 545 } 546 547 case Builtin::BI__builtin_ctzs: 548 case Builtin::BI__builtin_ctz: 549 case Builtin::BI__builtin_ctzl: 550 case Builtin::BI__builtin_ctzll: { 551 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 552 553 llvm::Type *ArgType = ArgValue->getType(); 554 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); 555 556 llvm::Type *ResultType = ConvertType(E->getType()); 557 Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); 558 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef}); 559 if (Result->getType() != ResultType) 560 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 561 "cast"); 562 return RValue::get(Result); 563 } 564 case Builtin::BI__builtin_clzs: 565 case Builtin::BI__builtin_clz: 566 case Builtin::BI__builtin_clzl: 567 case Builtin::BI__builtin_clzll: { 568 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 569 570 llvm::Type *ArgType = ArgValue->getType(); 571 Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType); 572 573 llvm::Type *ResultType = ConvertType(E->getType()); 574 Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); 575 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef}); 576 if (Result->getType() != ResultType) 577 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 578 "cast"); 579 return RValue::get(Result); 580 } 581 case Builtin::BI__builtin_ffs: 582 case Builtin::BI__builtin_ffsl: 583 case Builtin::BI__builtin_ffsll: { 584 // ffs(x) -> x ? cttz(x) + 1 : 0 585 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 586 587 llvm::Type *ArgType = ArgValue->getType(); 588 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); 589 590 llvm::Type *ResultType = ConvertType(E->getType()); 591 Value *Tmp = 592 Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}), 593 llvm::ConstantInt::get(ArgType, 1)); 594 Value *Zero = llvm::Constant::getNullValue(ArgType); 595 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero"); 596 Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs"); 597 if (Result->getType() != ResultType) 598 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 599 "cast"); 600 return RValue::get(Result); 601 } 602 case Builtin::BI__builtin_parity: 603 case Builtin::BI__builtin_parityl: 604 case Builtin::BI__builtin_parityll: { 605 // parity(x) -> ctpop(x) & 1 606 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 607 608 llvm::Type *ArgType = ArgValue->getType(); 609 Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType); 610 611 llvm::Type *ResultType = ConvertType(E->getType()); 612 Value *Tmp = Builder.CreateCall(F, ArgValue); 613 Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1)); 614 if (Result->getType() != ResultType) 615 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 616 "cast"); 617 return RValue::get(Result); 618 } 619 case Builtin::BI__builtin_popcount: 620 case Builtin::BI__builtin_popcountl: 621 case Builtin::BI__builtin_popcountll: { 622 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 623 624 llvm::Type *ArgType = ArgValue->getType(); 625 Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType); 626 627 llvm::Type *ResultType = ConvertType(E->getType()); 628 Value *Result = Builder.CreateCall(F, ArgValue); 629 if (Result->getType() != ResultType) 630 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 631 "cast"); 632 return RValue::get(Result); 633 } 634 case Builtin::BI__builtin_unpredictable: { 635 // Always return the argument of __builtin_unpredictable. LLVM does not 636 // handle this builtin. Metadata for this builtin should be added directly 637 // to instructions such as branches or switches that use it. 638 return RValue::get(EmitScalarExpr(E->getArg(0))); 639 } 640 case Builtin::BI__builtin_expect: { 641 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 642 llvm::Type *ArgType = ArgValue->getType(); 643 644 Value *ExpectedValue = EmitScalarExpr(E->getArg(1)); 645 // Don't generate llvm.expect on -O0 as the backend won't use it for 646 // anything. 647 // Note, we still IRGen ExpectedValue because it could have side-effects. 648 if (CGM.getCodeGenOpts().OptimizationLevel == 0) 649 return RValue::get(ArgValue); 650 651 Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType); 652 Value *Result = 653 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue}, "expval"); 654 return RValue::get(Result); 655 } 656 case Builtin::BI__builtin_assume_aligned: { 657 Value *PtrValue = EmitScalarExpr(E->getArg(0)); 658 Value *OffsetValue = 659 (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : nullptr; 660 661 Value *AlignmentValue = EmitScalarExpr(E->getArg(1)); 662 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue); 663 unsigned Alignment = (unsigned) AlignmentCI->getZExtValue(); 664 665 EmitAlignmentAssumption(PtrValue, Alignment, OffsetValue); 666 return RValue::get(PtrValue); 667 } 668 case Builtin::BI__assume: 669 case Builtin::BI__builtin_assume: { 670 if (E->getArg(0)->HasSideEffects(getContext())) 671 return RValue::get(nullptr); 672 673 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 674 Value *FnAssume = CGM.getIntrinsic(Intrinsic::assume); 675 return RValue::get(Builder.CreateCall(FnAssume, ArgValue)); 676 } 677 case Builtin::BI__builtin_bswap16: 678 case Builtin::BI__builtin_bswap32: 679 case Builtin::BI__builtin_bswap64: { 680 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bswap)); 681 } 682 case Builtin::BI__builtin_bitreverse8: 683 case Builtin::BI__builtin_bitreverse16: 684 case Builtin::BI__builtin_bitreverse32: 685 case Builtin::BI__builtin_bitreverse64: { 686 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bitreverse)); 687 } 688 case Builtin::BI__builtin_object_size: { 689 unsigned Type = 690 E->getArg(1)->EvaluateKnownConstInt(getContext()).getZExtValue(); 691 auto *ResType = cast<llvm::IntegerType>(ConvertType(E->getType())); 692 693 // We pass this builtin onto the optimizer so that it can figure out the 694 // object size in more complex cases. 695 return RValue::get(emitBuiltinObjectSize(E->getArg(0), Type, ResType)); 696 } 697 case Builtin::BI__builtin_prefetch: { 698 Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0)); 699 // FIXME: Technically these constants should of type 'int', yes? 700 RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) : 701 llvm::ConstantInt::get(Int32Ty, 0); 702 Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : 703 llvm::ConstantInt::get(Int32Ty, 3); 704 Value *Data = llvm::ConstantInt::get(Int32Ty, 1); 705 Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 706 return RValue::get(Builder.CreateCall(F, {Address, RW, Locality, Data})); 707 } 708 case Builtin::BI__builtin_readcyclecounter: { 709 Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter); 710 return RValue::get(Builder.CreateCall(F)); 711 } 712 case Builtin::BI__builtin___clear_cache: { 713 Value *Begin = EmitScalarExpr(E->getArg(0)); 714 Value *End = EmitScalarExpr(E->getArg(1)); 715 Value *F = CGM.getIntrinsic(Intrinsic::clear_cache); 716 return RValue::get(Builder.CreateCall(F, {Begin, End})); 717 } 718 case Builtin::BI__builtin_trap: 719 return RValue::get(EmitTrapCall(Intrinsic::trap)); 720 case Builtin::BI__debugbreak: 721 return RValue::get(EmitTrapCall(Intrinsic::debugtrap)); 722 case Builtin::BI__builtin_unreachable: { 723 if (SanOpts.has(SanitizerKind::Unreachable)) { 724 SanitizerScope SanScope(this); 725 EmitCheck(std::make_pair(static_cast<llvm::Value *>(Builder.getFalse()), 726 SanitizerKind::Unreachable), 727 "builtin_unreachable", EmitCheckSourceLocation(E->getExprLoc()), 728 None); 729 } else 730 Builder.CreateUnreachable(); 731 732 // We do need to preserve an insertion point. 733 EmitBlock(createBasicBlock("unreachable.cont")); 734 735 return RValue::get(nullptr); 736 } 737 738 case Builtin::BI__builtin_powi: 739 case Builtin::BI__builtin_powif: 740 case Builtin::BI__builtin_powil: { 741 Value *Base = EmitScalarExpr(E->getArg(0)); 742 Value *Exponent = EmitScalarExpr(E->getArg(1)); 743 llvm::Type *ArgType = Base->getType(); 744 Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType); 745 return RValue::get(Builder.CreateCall(F, {Base, Exponent})); 746 } 747 748 case Builtin::BI__builtin_isgreater: 749 case Builtin::BI__builtin_isgreaterequal: 750 case Builtin::BI__builtin_isless: 751 case Builtin::BI__builtin_islessequal: 752 case Builtin::BI__builtin_islessgreater: 753 case Builtin::BI__builtin_isunordered: { 754 // Ordered comparisons: we know the arguments to these are matching scalar 755 // floating point values. 756 Value *LHS = EmitScalarExpr(E->getArg(0)); 757 Value *RHS = EmitScalarExpr(E->getArg(1)); 758 759 switch (BuiltinID) { 760 default: llvm_unreachable("Unknown ordered comparison"); 761 case Builtin::BI__builtin_isgreater: 762 LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp"); 763 break; 764 case Builtin::BI__builtin_isgreaterequal: 765 LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp"); 766 break; 767 case Builtin::BI__builtin_isless: 768 LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp"); 769 break; 770 case Builtin::BI__builtin_islessequal: 771 LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp"); 772 break; 773 case Builtin::BI__builtin_islessgreater: 774 LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp"); 775 break; 776 case Builtin::BI__builtin_isunordered: 777 LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp"); 778 break; 779 } 780 // ZExt bool to int type. 781 return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType()))); 782 } 783 case Builtin::BI__builtin_isnan: { 784 Value *V = EmitScalarExpr(E->getArg(0)); 785 V = Builder.CreateFCmpUNO(V, V, "cmp"); 786 return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); 787 } 788 789 case Builtin::BI__builtin_isinf: 790 case Builtin::BI__builtin_isfinite: { 791 // isinf(x) --> fabs(x) == infinity 792 // isfinite(x) --> fabs(x) != infinity 793 // x != NaN via the ordered compare in either case. 794 Value *V = EmitScalarExpr(E->getArg(0)); 795 Value *Fabs = EmitFAbs(*this, V); 796 Constant *Infinity = ConstantFP::getInfinity(V->getType()); 797 CmpInst::Predicate Pred = (BuiltinID == Builtin::BI__builtin_isinf) 798 ? CmpInst::FCMP_OEQ 799 : CmpInst::FCMP_ONE; 800 Value *FCmp = Builder.CreateFCmp(Pred, Fabs, Infinity, "cmpinf"); 801 return RValue::get(Builder.CreateZExt(FCmp, ConvertType(E->getType()))); 802 } 803 804 case Builtin::BI__builtin_isinf_sign: { 805 // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0 806 Value *Arg = EmitScalarExpr(E->getArg(0)); 807 Value *AbsArg = EmitFAbs(*this, Arg); 808 Value *IsInf = Builder.CreateFCmpOEQ( 809 AbsArg, ConstantFP::getInfinity(Arg->getType()), "isinf"); 810 Value *IsNeg = EmitSignBit(*this, Arg); 811 812 llvm::Type *IntTy = ConvertType(E->getType()); 813 Value *Zero = Constant::getNullValue(IntTy); 814 Value *One = ConstantInt::get(IntTy, 1); 815 Value *NegativeOne = ConstantInt::get(IntTy, -1); 816 Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One); 817 Value *Result = Builder.CreateSelect(IsInf, SignResult, Zero); 818 return RValue::get(Result); 819 } 820 821 case Builtin::BI__builtin_isnormal: { 822 // isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min 823 Value *V = EmitScalarExpr(E->getArg(0)); 824 Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq"); 825 826 Value *Abs = EmitFAbs(*this, V); 827 Value *IsLessThanInf = 828 Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),"isinf"); 829 APFloat Smallest = APFloat::getSmallestNormalized( 830 getContext().getFloatTypeSemantics(E->getArg(0)->getType())); 831 Value *IsNormal = 832 Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest), 833 "isnormal"); 834 V = Builder.CreateAnd(Eq, IsLessThanInf, "and"); 835 V = Builder.CreateAnd(V, IsNormal, "and"); 836 return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); 837 } 838 839 case Builtin::BI__builtin_fpclassify: { 840 Value *V = EmitScalarExpr(E->getArg(5)); 841 llvm::Type *Ty = ConvertType(E->getArg(5)->getType()); 842 843 // Create Result 844 BasicBlock *Begin = Builder.GetInsertBlock(); 845 BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn); 846 Builder.SetInsertPoint(End); 847 PHINode *Result = 848 Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4, 849 "fpclassify_result"); 850 851 // if (V==0) return FP_ZERO 852 Builder.SetInsertPoint(Begin); 853 Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty), 854 "iszero"); 855 Value *ZeroLiteral = EmitScalarExpr(E->getArg(4)); 856 BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn); 857 Builder.CreateCondBr(IsZero, End, NotZero); 858 Result->addIncoming(ZeroLiteral, Begin); 859 860 // if (V != V) return FP_NAN 861 Builder.SetInsertPoint(NotZero); 862 Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp"); 863 Value *NanLiteral = EmitScalarExpr(E->getArg(0)); 864 BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn); 865 Builder.CreateCondBr(IsNan, End, NotNan); 866 Result->addIncoming(NanLiteral, NotZero); 867 868 // if (fabs(V) == infinity) return FP_INFINITY 869 Builder.SetInsertPoint(NotNan); 870 Value *VAbs = EmitFAbs(*this, V); 871 Value *IsInf = 872 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()), 873 "isinf"); 874 Value *InfLiteral = EmitScalarExpr(E->getArg(1)); 875 BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn); 876 Builder.CreateCondBr(IsInf, End, NotInf); 877 Result->addIncoming(InfLiteral, NotNan); 878 879 // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL 880 Builder.SetInsertPoint(NotInf); 881 APFloat Smallest = APFloat::getSmallestNormalized( 882 getContext().getFloatTypeSemantics(E->getArg(5)->getType())); 883 Value *IsNormal = 884 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest), 885 "isnormal"); 886 Value *NormalResult = 887 Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)), 888 EmitScalarExpr(E->getArg(3))); 889 Builder.CreateBr(End); 890 Result->addIncoming(NormalResult, NotInf); 891 892 // return Result 893 Builder.SetInsertPoint(End); 894 return RValue::get(Result); 895 } 896 897 case Builtin::BIalloca: 898 case Builtin::BI_alloca: 899 case Builtin::BI__builtin_alloca: { 900 Value *Size = EmitScalarExpr(E->getArg(0)); 901 return RValue::get(Builder.CreateAlloca(Builder.getInt8Ty(), Size)); 902 } 903 case Builtin::BIbzero: 904 case Builtin::BI__builtin_bzero: { 905 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 906 Value *SizeVal = EmitScalarExpr(E->getArg(1)); 907 EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), 908 E->getArg(0)->getExprLoc(), FD, 0); 909 Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false); 910 return RValue::get(Dest.getPointer()); 911 } 912 case Builtin::BImemcpy: 913 case Builtin::BI__builtin_memcpy: { 914 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 915 Address Src = EmitPointerWithAlignment(E->getArg(1)); 916 Value *SizeVal = EmitScalarExpr(E->getArg(2)); 917 EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), 918 E->getArg(0)->getExprLoc(), FD, 0); 919 EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(), 920 E->getArg(1)->getExprLoc(), FD, 1); 921 Builder.CreateMemCpy(Dest, Src, SizeVal, false); 922 return RValue::get(Dest.getPointer()); 923 } 924 925 case Builtin::BI__builtin___memcpy_chk: { 926 // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2. 927 llvm::APSInt Size, DstSize; 928 if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 929 !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 930 break; 931 if (Size.ugt(DstSize)) 932 break; 933 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 934 Address Src = EmitPointerWithAlignment(E->getArg(1)); 935 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 936 Builder.CreateMemCpy(Dest, Src, SizeVal, false); 937 return RValue::get(Dest.getPointer()); 938 } 939 940 case Builtin::BI__builtin_objc_memmove_collectable: { 941 Address DestAddr = EmitPointerWithAlignment(E->getArg(0)); 942 Address SrcAddr = EmitPointerWithAlignment(E->getArg(1)); 943 Value *SizeVal = EmitScalarExpr(E->getArg(2)); 944 CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, 945 DestAddr, SrcAddr, SizeVal); 946 return RValue::get(DestAddr.getPointer()); 947 } 948 949 case Builtin::BI__builtin___memmove_chk: { 950 // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2. 951 llvm::APSInt Size, DstSize; 952 if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 953 !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 954 break; 955 if (Size.ugt(DstSize)) 956 break; 957 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 958 Address Src = EmitPointerWithAlignment(E->getArg(1)); 959 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 960 Builder.CreateMemMove(Dest, Src, SizeVal, false); 961 return RValue::get(Dest.getPointer()); 962 } 963 964 case Builtin::BImemmove: 965 case Builtin::BI__builtin_memmove: { 966 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 967 Address Src = EmitPointerWithAlignment(E->getArg(1)); 968 Value *SizeVal = EmitScalarExpr(E->getArg(2)); 969 EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), 970 E->getArg(0)->getExprLoc(), FD, 0); 971 EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(), 972 E->getArg(1)->getExprLoc(), FD, 1); 973 Builder.CreateMemMove(Dest, Src, SizeVal, false); 974 return RValue::get(Dest.getPointer()); 975 } 976 case Builtin::BImemset: 977 case Builtin::BI__builtin_memset: { 978 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 979 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), 980 Builder.getInt8Ty()); 981 Value *SizeVal = EmitScalarExpr(E->getArg(2)); 982 EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), 983 E->getArg(0)->getExprLoc(), FD, 0); 984 Builder.CreateMemSet(Dest, ByteVal, SizeVal, false); 985 return RValue::get(Dest.getPointer()); 986 } 987 case Builtin::BI__builtin___memset_chk: { 988 // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2. 989 llvm::APSInt Size, DstSize; 990 if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 991 !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 992 break; 993 if (Size.ugt(DstSize)) 994 break; 995 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 996 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), 997 Builder.getInt8Ty()); 998 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 999 Builder.CreateMemSet(Dest, ByteVal, SizeVal, false); 1000 return RValue::get(Dest.getPointer()); 1001 } 1002 case Builtin::BI__builtin_dwarf_cfa: { 1003 // The offset in bytes from the first argument to the CFA. 1004 // 1005 // Why on earth is this in the frontend? Is there any reason at 1006 // all that the backend can't reasonably determine this while 1007 // lowering llvm.eh.dwarf.cfa()? 1008 // 1009 // TODO: If there's a satisfactory reason, add a target hook for 1010 // this instead of hard-coding 0, which is correct for most targets. 1011 int32_t Offset = 0; 1012 1013 Value *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa); 1014 return RValue::get(Builder.CreateCall(F, 1015 llvm::ConstantInt::get(Int32Ty, Offset))); 1016 } 1017 case Builtin::BI__builtin_return_address: { 1018 Value *Depth = 1019 CGM.EmitConstantExpr(E->getArg(0), getContext().UnsignedIntTy, this); 1020 Value *F = CGM.getIntrinsic(Intrinsic::returnaddress); 1021 return RValue::get(Builder.CreateCall(F, Depth)); 1022 } 1023 case Builtin::BI__builtin_frame_address: { 1024 Value *Depth = 1025 CGM.EmitConstantExpr(E->getArg(0), getContext().UnsignedIntTy, this); 1026 Value *F = CGM.getIntrinsic(Intrinsic::frameaddress); 1027 return RValue::get(Builder.CreateCall(F, Depth)); 1028 } 1029 case Builtin::BI__builtin_extract_return_addr: { 1030 Value *Address = EmitScalarExpr(E->getArg(0)); 1031 Value *Result = getTargetHooks().decodeReturnAddress(*this, Address); 1032 return RValue::get(Result); 1033 } 1034 case Builtin::BI__builtin_frob_return_addr: { 1035 Value *Address = EmitScalarExpr(E->getArg(0)); 1036 Value *Result = getTargetHooks().encodeReturnAddress(*this, Address); 1037 return RValue::get(Result); 1038 } 1039 case Builtin::BI__builtin_dwarf_sp_column: { 1040 llvm::IntegerType *Ty 1041 = cast<llvm::IntegerType>(ConvertType(E->getType())); 1042 int Column = getTargetHooks().getDwarfEHStackPointer(CGM); 1043 if (Column == -1) { 1044 CGM.ErrorUnsupported(E, "__builtin_dwarf_sp_column"); 1045 return RValue::get(llvm::UndefValue::get(Ty)); 1046 } 1047 return RValue::get(llvm::ConstantInt::get(Ty, Column, true)); 1048 } 1049 case Builtin::BI__builtin_init_dwarf_reg_size_table: { 1050 Value *Address = EmitScalarExpr(E->getArg(0)); 1051 if (getTargetHooks().initDwarfEHRegSizeTable(*this, Address)) 1052 CGM.ErrorUnsupported(E, "__builtin_init_dwarf_reg_size_table"); 1053 return RValue::get(llvm::UndefValue::get(ConvertType(E->getType()))); 1054 } 1055 case Builtin::BI__builtin_eh_return: { 1056 Value *Int = EmitScalarExpr(E->getArg(0)); 1057 Value *Ptr = EmitScalarExpr(E->getArg(1)); 1058 1059 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType()); 1060 assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) && 1061 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants"); 1062 Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32 1063 ? Intrinsic::eh_return_i32 1064 : Intrinsic::eh_return_i64); 1065 Builder.CreateCall(F, {Int, Ptr}); 1066 Builder.CreateUnreachable(); 1067 1068 // We do need to preserve an insertion point. 1069 EmitBlock(createBasicBlock("builtin_eh_return.cont")); 1070 1071 return RValue::get(nullptr); 1072 } 1073 case Builtin::BI__builtin_unwind_init: { 1074 Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init); 1075 return RValue::get(Builder.CreateCall(F)); 1076 } 1077 case Builtin::BI__builtin_extend_pointer: { 1078 // Extends a pointer to the size of an _Unwind_Word, which is 1079 // uint64_t on all platforms. Generally this gets poked into a 1080 // register and eventually used as an address, so if the 1081 // addressing registers are wider than pointers and the platform 1082 // doesn't implicitly ignore high-order bits when doing 1083 // addressing, we need to make sure we zext / sext based on 1084 // the platform's expectations. 1085 // 1086 // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html 1087 1088 // Cast the pointer to intptr_t. 1089 Value *Ptr = EmitScalarExpr(E->getArg(0)); 1090 Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast"); 1091 1092 // If that's 64 bits, we're done. 1093 if (IntPtrTy->getBitWidth() == 64) 1094 return RValue::get(Result); 1095 1096 // Otherwise, ask the codegen data what to do. 1097 if (getTargetHooks().extendPointerWithSExt()) 1098 return RValue::get(Builder.CreateSExt(Result, Int64Ty, "extend.sext")); 1099 else 1100 return RValue::get(Builder.CreateZExt(Result, Int64Ty, "extend.zext")); 1101 } 1102 case Builtin::BI__builtin_setjmp: { 1103 // Buffer is a void**. 1104 Address Buf = EmitPointerWithAlignment(E->getArg(0)); 1105 1106 // Store the frame pointer to the setjmp buffer. 1107 Value *FrameAddr = 1108 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), 1109 ConstantInt::get(Int32Ty, 0)); 1110 Builder.CreateStore(FrameAddr, Buf); 1111 1112 // Store the stack pointer to the setjmp buffer. 1113 Value *StackAddr = 1114 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave)); 1115 Address StackSaveSlot = 1116 Builder.CreateConstInBoundsGEP(Buf, 2, getPointerSize()); 1117 Builder.CreateStore(StackAddr, StackSaveSlot); 1118 1119 // Call LLVM's EH setjmp, which is lightweight. 1120 Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp); 1121 Buf = Builder.CreateBitCast(Buf, Int8PtrTy); 1122 return RValue::get(Builder.CreateCall(F, Buf.getPointer())); 1123 } 1124 case Builtin::BI__builtin_longjmp: { 1125 Value *Buf = EmitScalarExpr(E->getArg(0)); 1126 Buf = Builder.CreateBitCast(Buf, Int8PtrTy); 1127 1128 // Call LLVM's EH longjmp, which is lightweight. 1129 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf); 1130 1131 // longjmp doesn't return; mark this as unreachable. 1132 Builder.CreateUnreachable(); 1133 1134 // We do need to preserve an insertion point. 1135 EmitBlock(createBasicBlock("longjmp.cont")); 1136 1137 return RValue::get(nullptr); 1138 } 1139 case Builtin::BI__sync_fetch_and_add: 1140 case Builtin::BI__sync_fetch_and_sub: 1141 case Builtin::BI__sync_fetch_and_or: 1142 case Builtin::BI__sync_fetch_and_and: 1143 case Builtin::BI__sync_fetch_and_xor: 1144 case Builtin::BI__sync_fetch_and_nand: 1145 case Builtin::BI__sync_add_and_fetch: 1146 case Builtin::BI__sync_sub_and_fetch: 1147 case Builtin::BI__sync_and_and_fetch: 1148 case Builtin::BI__sync_or_and_fetch: 1149 case Builtin::BI__sync_xor_and_fetch: 1150 case Builtin::BI__sync_nand_and_fetch: 1151 case Builtin::BI__sync_val_compare_and_swap: 1152 case Builtin::BI__sync_bool_compare_and_swap: 1153 case Builtin::BI__sync_lock_test_and_set: 1154 case Builtin::BI__sync_lock_release: 1155 case Builtin::BI__sync_swap: 1156 llvm_unreachable("Shouldn't make it through sema"); 1157 case Builtin::BI__sync_fetch_and_add_1: 1158 case Builtin::BI__sync_fetch_and_add_2: 1159 case Builtin::BI__sync_fetch_and_add_4: 1160 case Builtin::BI__sync_fetch_and_add_8: 1161 case Builtin::BI__sync_fetch_and_add_16: 1162 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Add, E); 1163 case Builtin::BI__sync_fetch_and_sub_1: 1164 case Builtin::BI__sync_fetch_and_sub_2: 1165 case Builtin::BI__sync_fetch_and_sub_4: 1166 case Builtin::BI__sync_fetch_and_sub_8: 1167 case Builtin::BI__sync_fetch_and_sub_16: 1168 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Sub, E); 1169 case Builtin::BI__sync_fetch_and_or_1: 1170 case Builtin::BI__sync_fetch_and_or_2: 1171 case Builtin::BI__sync_fetch_and_or_4: 1172 case Builtin::BI__sync_fetch_and_or_8: 1173 case Builtin::BI__sync_fetch_and_or_16: 1174 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Or, E); 1175 case Builtin::BI__sync_fetch_and_and_1: 1176 case Builtin::BI__sync_fetch_and_and_2: 1177 case Builtin::BI__sync_fetch_and_and_4: 1178 case Builtin::BI__sync_fetch_and_and_8: 1179 case Builtin::BI__sync_fetch_and_and_16: 1180 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::And, E); 1181 case Builtin::BI__sync_fetch_and_xor_1: 1182 case Builtin::BI__sync_fetch_and_xor_2: 1183 case Builtin::BI__sync_fetch_and_xor_4: 1184 case Builtin::BI__sync_fetch_and_xor_8: 1185 case Builtin::BI__sync_fetch_and_xor_16: 1186 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xor, E); 1187 case Builtin::BI__sync_fetch_and_nand_1: 1188 case Builtin::BI__sync_fetch_and_nand_2: 1189 case Builtin::BI__sync_fetch_and_nand_4: 1190 case Builtin::BI__sync_fetch_and_nand_8: 1191 case Builtin::BI__sync_fetch_and_nand_16: 1192 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Nand, E); 1193 1194 // Clang extensions: not overloaded yet. 1195 case Builtin::BI__sync_fetch_and_min: 1196 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Min, E); 1197 case Builtin::BI__sync_fetch_and_max: 1198 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Max, E); 1199 case Builtin::BI__sync_fetch_and_umin: 1200 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMin, E); 1201 case Builtin::BI__sync_fetch_and_umax: 1202 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMax, E); 1203 1204 case Builtin::BI__sync_add_and_fetch_1: 1205 case Builtin::BI__sync_add_and_fetch_2: 1206 case Builtin::BI__sync_add_and_fetch_4: 1207 case Builtin::BI__sync_add_and_fetch_8: 1208 case Builtin::BI__sync_add_and_fetch_16: 1209 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Add, E, 1210 llvm::Instruction::Add); 1211 case Builtin::BI__sync_sub_and_fetch_1: 1212 case Builtin::BI__sync_sub_and_fetch_2: 1213 case Builtin::BI__sync_sub_and_fetch_4: 1214 case Builtin::BI__sync_sub_and_fetch_8: 1215 case Builtin::BI__sync_sub_and_fetch_16: 1216 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Sub, E, 1217 llvm::Instruction::Sub); 1218 case Builtin::BI__sync_and_and_fetch_1: 1219 case Builtin::BI__sync_and_and_fetch_2: 1220 case Builtin::BI__sync_and_and_fetch_4: 1221 case Builtin::BI__sync_and_and_fetch_8: 1222 case Builtin::BI__sync_and_and_fetch_16: 1223 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::And, E, 1224 llvm::Instruction::And); 1225 case Builtin::BI__sync_or_and_fetch_1: 1226 case Builtin::BI__sync_or_and_fetch_2: 1227 case Builtin::BI__sync_or_and_fetch_4: 1228 case Builtin::BI__sync_or_and_fetch_8: 1229 case Builtin::BI__sync_or_and_fetch_16: 1230 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Or, E, 1231 llvm::Instruction::Or); 1232 case Builtin::BI__sync_xor_and_fetch_1: 1233 case Builtin::BI__sync_xor_and_fetch_2: 1234 case Builtin::BI__sync_xor_and_fetch_4: 1235 case Builtin::BI__sync_xor_and_fetch_8: 1236 case Builtin::BI__sync_xor_and_fetch_16: 1237 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Xor, E, 1238 llvm::Instruction::Xor); 1239 case Builtin::BI__sync_nand_and_fetch_1: 1240 case Builtin::BI__sync_nand_and_fetch_2: 1241 case Builtin::BI__sync_nand_and_fetch_4: 1242 case Builtin::BI__sync_nand_and_fetch_8: 1243 case Builtin::BI__sync_nand_and_fetch_16: 1244 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Nand, E, 1245 llvm::Instruction::And, true); 1246 1247 case Builtin::BI__sync_val_compare_and_swap_1: 1248 case Builtin::BI__sync_val_compare_and_swap_2: 1249 case Builtin::BI__sync_val_compare_and_swap_4: 1250 case Builtin::BI__sync_val_compare_and_swap_8: 1251 case Builtin::BI__sync_val_compare_and_swap_16: 1252 return RValue::get(MakeAtomicCmpXchgValue(*this, E, false)); 1253 1254 case Builtin::BI__sync_bool_compare_and_swap_1: 1255 case Builtin::BI__sync_bool_compare_and_swap_2: 1256 case Builtin::BI__sync_bool_compare_and_swap_4: 1257 case Builtin::BI__sync_bool_compare_and_swap_8: 1258 case Builtin::BI__sync_bool_compare_and_swap_16: 1259 return RValue::get(MakeAtomicCmpXchgValue(*this, E, true)); 1260 1261 case Builtin::BI__sync_swap_1: 1262 case Builtin::BI__sync_swap_2: 1263 case Builtin::BI__sync_swap_4: 1264 case Builtin::BI__sync_swap_8: 1265 case Builtin::BI__sync_swap_16: 1266 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); 1267 1268 case Builtin::BI__sync_lock_test_and_set_1: 1269 case Builtin::BI__sync_lock_test_and_set_2: 1270 case Builtin::BI__sync_lock_test_and_set_4: 1271 case Builtin::BI__sync_lock_test_and_set_8: 1272 case Builtin::BI__sync_lock_test_and_set_16: 1273 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); 1274 1275 case Builtin::BI__sync_lock_release_1: 1276 case Builtin::BI__sync_lock_release_2: 1277 case Builtin::BI__sync_lock_release_4: 1278 case Builtin::BI__sync_lock_release_8: 1279 case Builtin::BI__sync_lock_release_16: { 1280 Value *Ptr = EmitScalarExpr(E->getArg(0)); 1281 QualType ElTy = E->getArg(0)->getType()->getPointeeType(); 1282 CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy); 1283 llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(), 1284 StoreSize.getQuantity() * 8); 1285 Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo()); 1286 llvm::StoreInst *Store = 1287 Builder.CreateAlignedStore(llvm::Constant::getNullValue(ITy), Ptr, 1288 StoreSize); 1289 Store->setAtomic(llvm::AtomicOrdering::Release); 1290 return RValue::get(nullptr); 1291 } 1292 1293 case Builtin::BI__sync_synchronize: { 1294 // We assume this is supposed to correspond to a C++0x-style 1295 // sequentially-consistent fence (i.e. this is only usable for 1296 // synchonization, not device I/O or anything like that). This intrinsic 1297 // is really badly designed in the sense that in theory, there isn't 1298 // any way to safely use it... but in practice, it mostly works 1299 // to use it with non-atomic loads and stores to get acquire/release 1300 // semantics. 1301 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent); 1302 return RValue::get(nullptr); 1303 } 1304 1305 case Builtin::BI__builtin_nontemporal_load: 1306 return RValue::get(EmitNontemporalLoad(*this, E)); 1307 case Builtin::BI__builtin_nontemporal_store: 1308 return RValue::get(EmitNontemporalStore(*this, E)); 1309 case Builtin::BI__c11_atomic_is_lock_free: 1310 case Builtin::BI__atomic_is_lock_free: { 1311 // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the 1312 // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since 1313 // _Atomic(T) is always properly-aligned. 1314 const char *LibCallName = "__atomic_is_lock_free"; 1315 CallArgList Args; 1316 Args.add(RValue::get(EmitScalarExpr(E->getArg(0))), 1317 getContext().getSizeType()); 1318 if (BuiltinID == Builtin::BI__atomic_is_lock_free) 1319 Args.add(RValue::get(EmitScalarExpr(E->getArg(1))), 1320 getContext().VoidPtrTy); 1321 else 1322 Args.add(RValue::get(llvm::Constant::getNullValue(VoidPtrTy)), 1323 getContext().VoidPtrTy); 1324 const CGFunctionInfo &FuncInfo = 1325 CGM.getTypes().arrangeBuiltinFunctionCall(E->getType(), Args); 1326 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo); 1327 llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName); 1328 return EmitCall(FuncInfo, Func, ReturnValueSlot(), Args); 1329 } 1330 1331 case Builtin::BI__atomic_test_and_set: { 1332 // Look at the argument type to determine whether this is a volatile 1333 // operation. The parameter type is always volatile. 1334 QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType(); 1335 bool Volatile = 1336 PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified(); 1337 1338 Value *Ptr = EmitScalarExpr(E->getArg(0)); 1339 unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace(); 1340 Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace)); 1341 Value *NewVal = Builder.getInt8(1); 1342 Value *Order = EmitScalarExpr(E->getArg(1)); 1343 if (isa<llvm::ConstantInt>(Order)) { 1344 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1345 AtomicRMWInst *Result = nullptr; 1346 switch (ord) { 1347 case 0: // memory_order_relaxed 1348 default: // invalid order 1349 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal, 1350 llvm::AtomicOrdering::Monotonic); 1351 break; 1352 case 1: // memory_order_consume 1353 case 2: // memory_order_acquire 1354 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal, 1355 llvm::AtomicOrdering::Acquire); 1356 break; 1357 case 3: // memory_order_release 1358 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal, 1359 llvm::AtomicOrdering::Release); 1360 break; 1361 case 4: // memory_order_acq_rel 1362 1363 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal, 1364 llvm::AtomicOrdering::AcquireRelease); 1365 break; 1366 case 5: // memory_order_seq_cst 1367 Result = Builder.CreateAtomicRMW( 1368 llvm::AtomicRMWInst::Xchg, Ptr, NewVal, 1369 llvm::AtomicOrdering::SequentiallyConsistent); 1370 break; 1371 } 1372 Result->setVolatile(Volatile); 1373 return RValue::get(Builder.CreateIsNotNull(Result, "tobool")); 1374 } 1375 1376 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1377 1378 llvm::BasicBlock *BBs[5] = { 1379 createBasicBlock("monotonic", CurFn), 1380 createBasicBlock("acquire", CurFn), 1381 createBasicBlock("release", CurFn), 1382 createBasicBlock("acqrel", CurFn), 1383 createBasicBlock("seqcst", CurFn) 1384 }; 1385 llvm::AtomicOrdering Orders[5] = { 1386 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire, 1387 llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease, 1388 llvm::AtomicOrdering::SequentiallyConsistent}; 1389 1390 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1391 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]); 1392 1393 Builder.SetInsertPoint(ContBB); 1394 PHINode *Result = Builder.CreatePHI(Int8Ty, 5, "was_set"); 1395 1396 for (unsigned i = 0; i < 5; ++i) { 1397 Builder.SetInsertPoint(BBs[i]); 1398 AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 1399 Ptr, NewVal, Orders[i]); 1400 RMW->setVolatile(Volatile); 1401 Result->addIncoming(RMW, BBs[i]); 1402 Builder.CreateBr(ContBB); 1403 } 1404 1405 SI->addCase(Builder.getInt32(0), BBs[0]); 1406 SI->addCase(Builder.getInt32(1), BBs[1]); 1407 SI->addCase(Builder.getInt32(2), BBs[1]); 1408 SI->addCase(Builder.getInt32(3), BBs[2]); 1409 SI->addCase(Builder.getInt32(4), BBs[3]); 1410 SI->addCase(Builder.getInt32(5), BBs[4]); 1411 1412 Builder.SetInsertPoint(ContBB); 1413 return RValue::get(Builder.CreateIsNotNull(Result, "tobool")); 1414 } 1415 1416 case Builtin::BI__atomic_clear: { 1417 QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType(); 1418 bool Volatile = 1419 PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified(); 1420 1421 Address Ptr = EmitPointerWithAlignment(E->getArg(0)); 1422 unsigned AddrSpace = Ptr.getPointer()->getType()->getPointerAddressSpace(); 1423 Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace)); 1424 Value *NewVal = Builder.getInt8(0); 1425 Value *Order = EmitScalarExpr(E->getArg(1)); 1426 if (isa<llvm::ConstantInt>(Order)) { 1427 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1428 StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile); 1429 switch (ord) { 1430 case 0: // memory_order_relaxed 1431 default: // invalid order 1432 Store->setOrdering(llvm::AtomicOrdering::Monotonic); 1433 break; 1434 case 3: // memory_order_release 1435 Store->setOrdering(llvm::AtomicOrdering::Release); 1436 break; 1437 case 5: // memory_order_seq_cst 1438 Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent); 1439 break; 1440 } 1441 return RValue::get(nullptr); 1442 } 1443 1444 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1445 1446 llvm::BasicBlock *BBs[3] = { 1447 createBasicBlock("monotonic", CurFn), 1448 createBasicBlock("release", CurFn), 1449 createBasicBlock("seqcst", CurFn) 1450 }; 1451 llvm::AtomicOrdering Orders[3] = { 1452 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release, 1453 llvm::AtomicOrdering::SequentiallyConsistent}; 1454 1455 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1456 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]); 1457 1458 for (unsigned i = 0; i < 3; ++i) { 1459 Builder.SetInsertPoint(BBs[i]); 1460 StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile); 1461 Store->setOrdering(Orders[i]); 1462 Builder.CreateBr(ContBB); 1463 } 1464 1465 SI->addCase(Builder.getInt32(0), BBs[0]); 1466 SI->addCase(Builder.getInt32(3), BBs[1]); 1467 SI->addCase(Builder.getInt32(5), BBs[2]); 1468 1469 Builder.SetInsertPoint(ContBB); 1470 return RValue::get(nullptr); 1471 } 1472 1473 case Builtin::BI__atomic_thread_fence: 1474 case Builtin::BI__atomic_signal_fence: 1475 case Builtin::BI__c11_atomic_thread_fence: 1476 case Builtin::BI__c11_atomic_signal_fence: { 1477 llvm::SynchronizationScope Scope; 1478 if (BuiltinID == Builtin::BI__atomic_signal_fence || 1479 BuiltinID == Builtin::BI__c11_atomic_signal_fence) 1480 Scope = llvm::SingleThread; 1481 else 1482 Scope = llvm::CrossThread; 1483 Value *Order = EmitScalarExpr(E->getArg(0)); 1484 if (isa<llvm::ConstantInt>(Order)) { 1485 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1486 switch (ord) { 1487 case 0: // memory_order_relaxed 1488 default: // invalid order 1489 break; 1490 case 1: // memory_order_consume 1491 case 2: // memory_order_acquire 1492 Builder.CreateFence(llvm::AtomicOrdering::Acquire, Scope); 1493 break; 1494 case 3: // memory_order_release 1495 Builder.CreateFence(llvm::AtomicOrdering::Release, Scope); 1496 break; 1497 case 4: // memory_order_acq_rel 1498 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, Scope); 1499 break; 1500 case 5: // memory_order_seq_cst 1501 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, 1502 Scope); 1503 break; 1504 } 1505 return RValue::get(nullptr); 1506 } 1507 1508 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB; 1509 AcquireBB = createBasicBlock("acquire", CurFn); 1510 ReleaseBB = createBasicBlock("release", CurFn); 1511 AcqRelBB = createBasicBlock("acqrel", CurFn); 1512 SeqCstBB = createBasicBlock("seqcst", CurFn); 1513 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1514 1515 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1516 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB); 1517 1518 Builder.SetInsertPoint(AcquireBB); 1519 Builder.CreateFence(llvm::AtomicOrdering::Acquire, Scope); 1520 Builder.CreateBr(ContBB); 1521 SI->addCase(Builder.getInt32(1), AcquireBB); 1522 SI->addCase(Builder.getInt32(2), AcquireBB); 1523 1524 Builder.SetInsertPoint(ReleaseBB); 1525 Builder.CreateFence(llvm::AtomicOrdering::Release, Scope); 1526 Builder.CreateBr(ContBB); 1527 SI->addCase(Builder.getInt32(3), ReleaseBB); 1528 1529 Builder.SetInsertPoint(AcqRelBB); 1530 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, Scope); 1531 Builder.CreateBr(ContBB); 1532 SI->addCase(Builder.getInt32(4), AcqRelBB); 1533 1534 Builder.SetInsertPoint(SeqCstBB); 1535 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1536 Builder.CreateBr(ContBB); 1537 SI->addCase(Builder.getInt32(5), SeqCstBB); 1538 1539 Builder.SetInsertPoint(ContBB); 1540 return RValue::get(nullptr); 1541 } 1542 1543 // Library functions with special handling. 1544 case Builtin::BIsqrt: 1545 case Builtin::BIsqrtf: 1546 case Builtin::BIsqrtl: { 1547 // Transform a call to sqrt* into a @llvm.sqrt.* intrinsic call, but only 1548 // in finite- or unsafe-math mode (the intrinsic has different semantics 1549 // for handling negative numbers compared to the library function, so 1550 // -fmath-errno=0 is not enough). 1551 if (!FD->hasAttr<ConstAttr>()) 1552 break; 1553 if (!(CGM.getCodeGenOpts().UnsafeFPMath || 1554 CGM.getCodeGenOpts().NoNaNsFPMath)) 1555 break; 1556 Value *Arg0 = EmitScalarExpr(E->getArg(0)); 1557 llvm::Type *ArgType = Arg0->getType(); 1558 Value *F = CGM.getIntrinsic(Intrinsic::sqrt, ArgType); 1559 return RValue::get(Builder.CreateCall(F, Arg0)); 1560 } 1561 1562 case Builtin::BI__builtin_pow: 1563 case Builtin::BI__builtin_powf: 1564 case Builtin::BI__builtin_powl: 1565 case Builtin::BIpow: 1566 case Builtin::BIpowf: 1567 case Builtin::BIpowl: { 1568 // Transform a call to pow* into a @llvm.pow.* intrinsic call. 1569 if (!FD->hasAttr<ConstAttr>()) 1570 break; 1571 Value *Base = EmitScalarExpr(E->getArg(0)); 1572 Value *Exponent = EmitScalarExpr(E->getArg(1)); 1573 llvm::Type *ArgType = Base->getType(); 1574 Value *F = CGM.getIntrinsic(Intrinsic::pow, ArgType); 1575 return RValue::get(Builder.CreateCall(F, {Base, Exponent})); 1576 } 1577 1578 case Builtin::BIfma: 1579 case Builtin::BIfmaf: 1580 case Builtin::BIfmal: 1581 case Builtin::BI__builtin_fma: 1582 case Builtin::BI__builtin_fmaf: 1583 case Builtin::BI__builtin_fmal: { 1584 // Rewrite fma to intrinsic. 1585 Value *FirstArg = EmitScalarExpr(E->getArg(0)); 1586 llvm::Type *ArgType = FirstArg->getType(); 1587 Value *F = CGM.getIntrinsic(Intrinsic::fma, ArgType); 1588 return RValue::get( 1589 Builder.CreateCall(F, {FirstArg, EmitScalarExpr(E->getArg(1)), 1590 EmitScalarExpr(E->getArg(2))})); 1591 } 1592 1593 case Builtin::BI__builtin_signbit: 1594 case Builtin::BI__builtin_signbitf: 1595 case Builtin::BI__builtin_signbitl: { 1596 return RValue::get( 1597 Builder.CreateZExt(EmitSignBit(*this, EmitScalarExpr(E->getArg(0))), 1598 ConvertType(E->getType()))); 1599 } 1600 case Builtin::BI__builtin_annotation: { 1601 llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0)); 1602 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::annotation, 1603 AnnVal->getType()); 1604 1605 // Get the annotation string, go through casts. Sema requires this to be a 1606 // non-wide string literal, potentially casted, so the cast<> is safe. 1607 const Expr *AnnotationStrExpr = E->getArg(1)->IgnoreParenCasts(); 1608 StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString(); 1609 return RValue::get(EmitAnnotationCall(F, AnnVal, Str, E->getExprLoc())); 1610 } 1611 case Builtin::BI__builtin_addcb: 1612 case Builtin::BI__builtin_addcs: 1613 case Builtin::BI__builtin_addc: 1614 case Builtin::BI__builtin_addcl: 1615 case Builtin::BI__builtin_addcll: 1616 case Builtin::BI__builtin_subcb: 1617 case Builtin::BI__builtin_subcs: 1618 case Builtin::BI__builtin_subc: 1619 case Builtin::BI__builtin_subcl: 1620 case Builtin::BI__builtin_subcll: { 1621 1622 // We translate all of these builtins from expressions of the form: 1623 // int x = ..., y = ..., carryin = ..., carryout, result; 1624 // result = __builtin_addc(x, y, carryin, &carryout); 1625 // 1626 // to LLVM IR of the form: 1627 // 1628 // %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y) 1629 // %tmpsum1 = extractvalue {i32, i1} %tmp1, 0 1630 // %carry1 = extractvalue {i32, i1} %tmp1, 1 1631 // %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1, 1632 // i32 %carryin) 1633 // %result = extractvalue {i32, i1} %tmp2, 0 1634 // %carry2 = extractvalue {i32, i1} %tmp2, 1 1635 // %tmp3 = or i1 %carry1, %carry2 1636 // %tmp4 = zext i1 %tmp3 to i32 1637 // store i32 %tmp4, i32* %carryout 1638 1639 // Scalarize our inputs. 1640 llvm::Value *X = EmitScalarExpr(E->getArg(0)); 1641 llvm::Value *Y = EmitScalarExpr(E->getArg(1)); 1642 llvm::Value *Carryin = EmitScalarExpr(E->getArg(2)); 1643 Address CarryOutPtr = EmitPointerWithAlignment(E->getArg(3)); 1644 1645 // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow. 1646 llvm::Intrinsic::ID IntrinsicId; 1647 switch (BuiltinID) { 1648 default: llvm_unreachable("Unknown multiprecision builtin id."); 1649 case Builtin::BI__builtin_addcb: 1650 case Builtin::BI__builtin_addcs: 1651 case Builtin::BI__builtin_addc: 1652 case Builtin::BI__builtin_addcl: 1653 case Builtin::BI__builtin_addcll: 1654 IntrinsicId = llvm::Intrinsic::uadd_with_overflow; 1655 break; 1656 case Builtin::BI__builtin_subcb: 1657 case Builtin::BI__builtin_subcs: 1658 case Builtin::BI__builtin_subc: 1659 case Builtin::BI__builtin_subcl: 1660 case Builtin::BI__builtin_subcll: 1661 IntrinsicId = llvm::Intrinsic::usub_with_overflow; 1662 break; 1663 } 1664 1665 // Construct our resulting LLVM IR expression. 1666 llvm::Value *Carry1; 1667 llvm::Value *Sum1 = EmitOverflowIntrinsic(*this, IntrinsicId, 1668 X, Y, Carry1); 1669 llvm::Value *Carry2; 1670 llvm::Value *Sum2 = EmitOverflowIntrinsic(*this, IntrinsicId, 1671 Sum1, Carryin, Carry2); 1672 llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2), 1673 X->getType()); 1674 Builder.CreateStore(CarryOut, CarryOutPtr); 1675 return RValue::get(Sum2); 1676 } 1677 1678 case Builtin::BI__builtin_add_overflow: 1679 case Builtin::BI__builtin_sub_overflow: 1680 case Builtin::BI__builtin_mul_overflow: { 1681 const clang::Expr *LeftArg = E->getArg(0); 1682 const clang::Expr *RightArg = E->getArg(1); 1683 const clang::Expr *ResultArg = E->getArg(2); 1684 1685 clang::QualType ResultQTy = 1686 ResultArg->getType()->castAs<PointerType>()->getPointeeType(); 1687 1688 WidthAndSignedness LeftInfo = 1689 getIntegerWidthAndSignedness(CGM.getContext(), LeftArg->getType()); 1690 WidthAndSignedness RightInfo = 1691 getIntegerWidthAndSignedness(CGM.getContext(), RightArg->getType()); 1692 WidthAndSignedness ResultInfo = 1693 getIntegerWidthAndSignedness(CGM.getContext(), ResultQTy); 1694 WidthAndSignedness EncompassingInfo = 1695 EncompassingIntegerType({LeftInfo, RightInfo, ResultInfo}); 1696 1697 llvm::Type *EncompassingLLVMTy = 1698 llvm::IntegerType::get(CGM.getLLVMContext(), EncompassingInfo.Width); 1699 1700 llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(ResultQTy); 1701 1702 llvm::Intrinsic::ID IntrinsicId; 1703 switch (BuiltinID) { 1704 default: 1705 llvm_unreachable("Unknown overflow builtin id."); 1706 case Builtin::BI__builtin_add_overflow: 1707 IntrinsicId = EncompassingInfo.Signed 1708 ? llvm::Intrinsic::sadd_with_overflow 1709 : llvm::Intrinsic::uadd_with_overflow; 1710 break; 1711 case Builtin::BI__builtin_sub_overflow: 1712 IntrinsicId = EncompassingInfo.Signed 1713 ? llvm::Intrinsic::ssub_with_overflow 1714 : llvm::Intrinsic::usub_with_overflow; 1715 break; 1716 case Builtin::BI__builtin_mul_overflow: 1717 IntrinsicId = EncompassingInfo.Signed 1718 ? llvm::Intrinsic::smul_with_overflow 1719 : llvm::Intrinsic::umul_with_overflow; 1720 break; 1721 } 1722 1723 llvm::Value *Left = EmitScalarExpr(LeftArg); 1724 llvm::Value *Right = EmitScalarExpr(RightArg); 1725 Address ResultPtr = EmitPointerWithAlignment(ResultArg); 1726 1727 // Extend each operand to the encompassing type. 1728 Left = Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed); 1729 Right = Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed); 1730 1731 // Perform the operation on the extended values. 1732 llvm::Value *Overflow, *Result; 1733 Result = EmitOverflowIntrinsic(*this, IntrinsicId, Left, Right, Overflow); 1734 1735 if (EncompassingInfo.Width > ResultInfo.Width) { 1736 // The encompassing type is wider than the result type, so we need to 1737 // truncate it. 1738 llvm::Value *ResultTrunc = Builder.CreateTrunc(Result, ResultLLVMTy); 1739 1740 // To see if the truncation caused an overflow, we will extend 1741 // the result and then compare it to the original result. 1742 llvm::Value *ResultTruncExt = Builder.CreateIntCast( 1743 ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed); 1744 llvm::Value *TruncationOverflow = 1745 Builder.CreateICmpNE(Result, ResultTruncExt); 1746 1747 Overflow = Builder.CreateOr(Overflow, TruncationOverflow); 1748 Result = ResultTrunc; 1749 } 1750 1751 // Finally, store the result using the pointer. 1752 bool isVolatile = 1753 ResultArg->getType()->getPointeeType().isVolatileQualified(); 1754 Builder.CreateStore(EmitToMemory(Result, ResultQTy), ResultPtr, isVolatile); 1755 1756 return RValue::get(Overflow); 1757 } 1758 1759 case Builtin::BI__builtin_uadd_overflow: 1760 case Builtin::BI__builtin_uaddl_overflow: 1761 case Builtin::BI__builtin_uaddll_overflow: 1762 case Builtin::BI__builtin_usub_overflow: 1763 case Builtin::BI__builtin_usubl_overflow: 1764 case Builtin::BI__builtin_usubll_overflow: 1765 case Builtin::BI__builtin_umul_overflow: 1766 case Builtin::BI__builtin_umull_overflow: 1767 case Builtin::BI__builtin_umulll_overflow: 1768 case Builtin::BI__builtin_sadd_overflow: 1769 case Builtin::BI__builtin_saddl_overflow: 1770 case Builtin::BI__builtin_saddll_overflow: 1771 case Builtin::BI__builtin_ssub_overflow: 1772 case Builtin::BI__builtin_ssubl_overflow: 1773 case Builtin::BI__builtin_ssubll_overflow: 1774 case Builtin::BI__builtin_smul_overflow: 1775 case Builtin::BI__builtin_smull_overflow: 1776 case Builtin::BI__builtin_smulll_overflow: { 1777 1778 // We translate all of these builtins directly to the relevant llvm IR node. 1779 1780 // Scalarize our inputs. 1781 llvm::Value *X = EmitScalarExpr(E->getArg(0)); 1782 llvm::Value *Y = EmitScalarExpr(E->getArg(1)); 1783 Address SumOutPtr = EmitPointerWithAlignment(E->getArg(2)); 1784 1785 // Decide which of the overflow intrinsics we are lowering to: 1786 llvm::Intrinsic::ID IntrinsicId; 1787 switch (BuiltinID) { 1788 default: llvm_unreachable("Unknown overflow builtin id."); 1789 case Builtin::BI__builtin_uadd_overflow: 1790 case Builtin::BI__builtin_uaddl_overflow: 1791 case Builtin::BI__builtin_uaddll_overflow: 1792 IntrinsicId = llvm::Intrinsic::uadd_with_overflow; 1793 break; 1794 case Builtin::BI__builtin_usub_overflow: 1795 case Builtin::BI__builtin_usubl_overflow: 1796 case Builtin::BI__builtin_usubll_overflow: 1797 IntrinsicId = llvm::Intrinsic::usub_with_overflow; 1798 break; 1799 case Builtin::BI__builtin_umul_overflow: 1800 case Builtin::BI__builtin_umull_overflow: 1801 case Builtin::BI__builtin_umulll_overflow: 1802 IntrinsicId = llvm::Intrinsic::umul_with_overflow; 1803 break; 1804 case Builtin::BI__builtin_sadd_overflow: 1805 case Builtin::BI__builtin_saddl_overflow: 1806 case Builtin::BI__builtin_saddll_overflow: 1807 IntrinsicId = llvm::Intrinsic::sadd_with_overflow; 1808 break; 1809 case Builtin::BI__builtin_ssub_overflow: 1810 case Builtin::BI__builtin_ssubl_overflow: 1811 case Builtin::BI__builtin_ssubll_overflow: 1812 IntrinsicId = llvm::Intrinsic::ssub_with_overflow; 1813 break; 1814 case Builtin::BI__builtin_smul_overflow: 1815 case Builtin::BI__builtin_smull_overflow: 1816 case Builtin::BI__builtin_smulll_overflow: 1817 IntrinsicId = llvm::Intrinsic::smul_with_overflow; 1818 break; 1819 } 1820 1821 1822 llvm::Value *Carry; 1823 llvm::Value *Sum = EmitOverflowIntrinsic(*this, IntrinsicId, X, Y, Carry); 1824 Builder.CreateStore(Sum, SumOutPtr); 1825 1826 return RValue::get(Carry); 1827 } 1828 case Builtin::BI__builtin_addressof: 1829 return RValue::get(EmitLValue(E->getArg(0)).getPointer()); 1830 case Builtin::BI__builtin_operator_new: 1831 return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(), 1832 E->getArg(0), false); 1833 case Builtin::BI__builtin_operator_delete: 1834 return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(), 1835 E->getArg(0), true); 1836 case Builtin::BI__noop: 1837 // __noop always evaluates to an integer literal zero. 1838 return RValue::get(ConstantInt::get(IntTy, 0)); 1839 case Builtin::BI__builtin_call_with_static_chain: { 1840 const CallExpr *Call = cast<CallExpr>(E->getArg(0)); 1841 const Expr *Chain = E->getArg(1); 1842 return EmitCall(Call->getCallee()->getType(), 1843 EmitScalarExpr(Call->getCallee()), Call, ReturnValue, 1844 Call->getCalleeDecl(), EmitScalarExpr(Chain)); 1845 } 1846 case Builtin::BI_InterlockedExchange: 1847 case Builtin::BI_InterlockedExchangePointer: 1848 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); 1849 case Builtin::BI_InterlockedCompareExchangePointer: { 1850 llvm::Type *RTy; 1851 llvm::IntegerType *IntType = 1852 IntegerType::get(getLLVMContext(), 1853 getContext().getTypeSize(E->getType())); 1854 llvm::Type *IntPtrType = IntType->getPointerTo(); 1855 1856 llvm::Value *Destination = 1857 Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), IntPtrType); 1858 1859 llvm::Value *Exchange = EmitScalarExpr(E->getArg(1)); 1860 RTy = Exchange->getType(); 1861 Exchange = Builder.CreatePtrToInt(Exchange, IntType); 1862 1863 llvm::Value *Comparand = 1864 Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType); 1865 1866 auto Result = 1867 Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange, 1868 AtomicOrdering::SequentiallyConsistent, 1869 AtomicOrdering::SequentiallyConsistent); 1870 Result->setVolatile(true); 1871 1872 return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result, 1873 0), 1874 RTy)); 1875 } 1876 case Builtin::BI_InterlockedCompareExchange: { 1877 AtomicCmpXchgInst *CXI = Builder.CreateAtomicCmpXchg( 1878 EmitScalarExpr(E->getArg(0)), 1879 EmitScalarExpr(E->getArg(2)), 1880 EmitScalarExpr(E->getArg(1)), 1881 AtomicOrdering::SequentiallyConsistent, 1882 AtomicOrdering::SequentiallyConsistent); 1883 CXI->setVolatile(true); 1884 return RValue::get(Builder.CreateExtractValue(CXI, 0)); 1885 } 1886 case Builtin::BI_InterlockedIncrement: { 1887 llvm::Type *IntTy = ConvertType(E->getType()); 1888 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1889 AtomicRMWInst::Add, 1890 EmitScalarExpr(E->getArg(0)), 1891 ConstantInt::get(IntTy, 1), 1892 llvm::AtomicOrdering::SequentiallyConsistent); 1893 RMWI->setVolatile(true); 1894 return RValue::get(Builder.CreateAdd(RMWI, ConstantInt::get(IntTy, 1))); 1895 } 1896 case Builtin::BI_InterlockedDecrement: { 1897 llvm::Type *IntTy = ConvertType(E->getType()); 1898 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1899 AtomicRMWInst::Sub, 1900 EmitScalarExpr(E->getArg(0)), 1901 ConstantInt::get(IntTy, 1), 1902 llvm::AtomicOrdering::SequentiallyConsistent); 1903 RMWI->setVolatile(true); 1904 return RValue::get(Builder.CreateSub(RMWI, ConstantInt::get(IntTy, 1))); 1905 } 1906 case Builtin::BI_InterlockedExchangeAdd: { 1907 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1908 AtomicRMWInst::Add, 1909 EmitScalarExpr(E->getArg(0)), 1910 EmitScalarExpr(E->getArg(1)), 1911 llvm::AtomicOrdering::SequentiallyConsistent); 1912 RMWI->setVolatile(true); 1913 return RValue::get(RMWI); 1914 } 1915 case Builtin::BI__readfsdword: { 1916 llvm::Type *IntTy = ConvertType(E->getType()); 1917 Value *IntToPtr = 1918 Builder.CreateIntToPtr(EmitScalarExpr(E->getArg(0)), 1919 llvm::PointerType::get(IntTy, 257)); 1920 LoadInst *Load = 1921 Builder.CreateDefaultAlignedLoad(IntToPtr, /*isVolatile=*/true); 1922 return RValue::get(Load); 1923 } 1924 1925 case Builtin::BI__exception_code: 1926 case Builtin::BI_exception_code: 1927 return RValue::get(EmitSEHExceptionCode()); 1928 case Builtin::BI__exception_info: 1929 case Builtin::BI_exception_info: 1930 return RValue::get(EmitSEHExceptionInfo()); 1931 case Builtin::BI__abnormal_termination: 1932 case Builtin::BI_abnormal_termination: 1933 return RValue::get(EmitSEHAbnormalTermination()); 1934 case Builtin::BI_setjmpex: { 1935 if (getTarget().getTriple().isOSMSVCRT()) { 1936 llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy}; 1937 llvm::AttributeSet ReturnsTwiceAttr = 1938 AttributeSet::get(getLLVMContext(), llvm::AttributeSet::FunctionIndex, 1939 llvm::Attribute::ReturnsTwice); 1940 llvm::Constant *SetJmpEx = CGM.CreateRuntimeFunction( 1941 llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false), 1942 "_setjmpex", ReturnsTwiceAttr); 1943 llvm::Value *Buf = Builder.CreateBitOrPointerCast( 1944 EmitScalarExpr(E->getArg(0)), Int8PtrTy); 1945 llvm::Value *FrameAddr = 1946 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), 1947 ConstantInt::get(Int32Ty, 0)); 1948 llvm::Value *Args[] = {Buf, FrameAddr}; 1949 llvm::CallSite CS = EmitRuntimeCallOrInvoke(SetJmpEx, Args); 1950 CS.setAttributes(ReturnsTwiceAttr); 1951 return RValue::get(CS.getInstruction()); 1952 } 1953 break; 1954 } 1955 case Builtin::BI_setjmp: { 1956 if (getTarget().getTriple().isOSMSVCRT()) { 1957 llvm::AttributeSet ReturnsTwiceAttr = 1958 AttributeSet::get(getLLVMContext(), llvm::AttributeSet::FunctionIndex, 1959 llvm::Attribute::ReturnsTwice); 1960 llvm::Value *Buf = Builder.CreateBitOrPointerCast( 1961 EmitScalarExpr(E->getArg(0)), Int8PtrTy); 1962 llvm::CallSite CS; 1963 if (getTarget().getTriple().getArch() == llvm::Triple::x86) { 1964 llvm::Type *ArgTypes[] = {Int8PtrTy, IntTy}; 1965 llvm::Constant *SetJmp3 = CGM.CreateRuntimeFunction( 1966 llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/true), 1967 "_setjmp3", ReturnsTwiceAttr); 1968 llvm::Value *Count = ConstantInt::get(IntTy, 0); 1969 llvm::Value *Args[] = {Buf, Count}; 1970 CS = EmitRuntimeCallOrInvoke(SetJmp3, Args); 1971 } else { 1972 llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy}; 1973 llvm::Constant *SetJmp = CGM.CreateRuntimeFunction( 1974 llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false), 1975 "_setjmp", ReturnsTwiceAttr); 1976 llvm::Value *FrameAddr = 1977 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), 1978 ConstantInt::get(Int32Ty, 0)); 1979 llvm::Value *Args[] = {Buf, FrameAddr}; 1980 CS = EmitRuntimeCallOrInvoke(SetJmp, Args); 1981 } 1982 CS.setAttributes(ReturnsTwiceAttr); 1983 return RValue::get(CS.getInstruction()); 1984 } 1985 break; 1986 } 1987 1988 case Builtin::BI__GetExceptionInfo: { 1989 if (llvm::GlobalVariable *GV = 1990 CGM.getCXXABI().getThrowInfo(FD->getParamDecl(0)->getType())) 1991 return RValue::get(llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy)); 1992 break; 1993 } 1994 1995 // OpenCL v2.0 s6.13.16.2, Built-in pipe read and write functions 1996 case Builtin::BIread_pipe: 1997 case Builtin::BIwrite_pipe: { 1998 Value *Arg0 = EmitScalarExpr(E->getArg(0)), 1999 *Arg1 = EmitScalarExpr(E->getArg(1)); 2000 2001 // Type of the generic packet parameter. 2002 unsigned GenericAS = 2003 getContext().getTargetAddressSpace(LangAS::opencl_generic); 2004 llvm::Type *I8PTy = llvm::PointerType::get( 2005 llvm::Type::getInt8Ty(getLLVMContext()), GenericAS); 2006 2007 // Testing which overloaded version we should generate the call for. 2008 if (2U == E->getNumArgs()) { 2009 const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_2" 2010 : "__write_pipe_2"; 2011 // Creating a generic function type to be able to call with any builtin or 2012 // user defined type. 2013 llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy}; 2014 llvm::FunctionType *FTy = llvm::FunctionType::get( 2015 Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false); 2016 Value *BCast = Builder.CreatePointerCast(Arg1, I8PTy); 2017 return RValue::get(Builder.CreateCall( 2018 CGM.CreateRuntimeFunction(FTy, Name), {Arg0, BCast})); 2019 } else { 2020 assert(4 == E->getNumArgs() && 2021 "Illegal number of parameters to pipe function"); 2022 const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_4" 2023 : "__write_pipe_4"; 2024 2025 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy}; 2026 Value *Arg2 = EmitScalarExpr(E->getArg(2)), 2027 *Arg3 = EmitScalarExpr(E->getArg(3)); 2028 llvm::FunctionType *FTy = llvm::FunctionType::get( 2029 Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false); 2030 Value *BCast = Builder.CreatePointerCast(Arg3, I8PTy); 2031 // We know the third argument is an integer type, but we may need to cast 2032 // it to i32. 2033 if (Arg2->getType() != Int32Ty) 2034 Arg2 = Builder.CreateZExtOrTrunc(Arg2, Int32Ty); 2035 return RValue::get(Builder.CreateCall( 2036 CGM.CreateRuntimeFunction(FTy, Name), {Arg0, Arg1, Arg2, BCast})); 2037 } 2038 } 2039 // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe reserve read and write 2040 // functions 2041 case Builtin::BIreserve_read_pipe: 2042 case Builtin::BIreserve_write_pipe: 2043 case Builtin::BIwork_group_reserve_read_pipe: 2044 case Builtin::BIwork_group_reserve_write_pipe: 2045 case Builtin::BIsub_group_reserve_read_pipe: 2046 case Builtin::BIsub_group_reserve_write_pipe: { 2047 // Composing the mangled name for the function. 2048 const char *Name; 2049 if (BuiltinID == Builtin::BIreserve_read_pipe) 2050 Name = "__reserve_read_pipe"; 2051 else if (BuiltinID == Builtin::BIreserve_write_pipe) 2052 Name = "__reserve_write_pipe"; 2053 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe) 2054 Name = "__work_group_reserve_read_pipe"; 2055 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe) 2056 Name = "__work_group_reserve_write_pipe"; 2057 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe) 2058 Name = "__sub_group_reserve_read_pipe"; 2059 else 2060 Name = "__sub_group_reserve_write_pipe"; 2061 2062 Value *Arg0 = EmitScalarExpr(E->getArg(0)), 2063 *Arg1 = EmitScalarExpr(E->getArg(1)); 2064 llvm::Type *ReservedIDTy = ConvertType(getContext().OCLReserveIDTy); 2065 2066 // Building the generic function prototype. 2067 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty}; 2068 llvm::FunctionType *FTy = llvm::FunctionType::get( 2069 ReservedIDTy, llvm::ArrayRef<llvm::Type *>(ArgTys), false); 2070 // We know the second argument is an integer type, but we may need to cast 2071 // it to i32. 2072 if (Arg1->getType() != Int32Ty) 2073 Arg1 = Builder.CreateZExtOrTrunc(Arg1, Int32Ty); 2074 return RValue::get( 2075 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), {Arg0, Arg1})); 2076 } 2077 // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe commit read and write 2078 // functions 2079 case Builtin::BIcommit_read_pipe: 2080 case Builtin::BIcommit_write_pipe: 2081 case Builtin::BIwork_group_commit_read_pipe: 2082 case Builtin::BIwork_group_commit_write_pipe: 2083 case Builtin::BIsub_group_commit_read_pipe: 2084 case Builtin::BIsub_group_commit_write_pipe: { 2085 const char *Name; 2086 if (BuiltinID == Builtin::BIcommit_read_pipe) 2087 Name = "__commit_read_pipe"; 2088 else if (BuiltinID == Builtin::BIcommit_write_pipe) 2089 Name = "__commit_write_pipe"; 2090 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe) 2091 Name = "__work_group_commit_read_pipe"; 2092 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe) 2093 Name = "__work_group_commit_write_pipe"; 2094 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe) 2095 Name = "__sub_group_commit_read_pipe"; 2096 else 2097 Name = "__sub_group_commit_write_pipe"; 2098 2099 Value *Arg0 = EmitScalarExpr(E->getArg(0)), 2100 *Arg1 = EmitScalarExpr(E->getArg(1)); 2101 2102 // Building the generic function prototype. 2103 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType()}; 2104 llvm::FunctionType *FTy = 2105 llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()), 2106 llvm::ArrayRef<llvm::Type *>(ArgTys), false); 2107 2108 return RValue::get( 2109 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), {Arg0, Arg1})); 2110 } 2111 // OpenCL v2.0 s6.13.16.4 Built-in pipe query functions 2112 case Builtin::BIget_pipe_num_packets: 2113 case Builtin::BIget_pipe_max_packets: { 2114 const char *Name; 2115 if (BuiltinID == Builtin::BIget_pipe_num_packets) 2116 Name = "__get_pipe_num_packets"; 2117 else 2118 Name = "__get_pipe_max_packets"; 2119 2120 // Building the generic function prototype. 2121 Value *Arg0 = EmitScalarExpr(E->getArg(0)); 2122 llvm::Type *ArgTys[] = {Arg0->getType()}; 2123 llvm::FunctionType *FTy = llvm::FunctionType::get( 2124 Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false); 2125 2126 return RValue::get( 2127 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), {Arg0})); 2128 } 2129 2130 // OpenCL v2.0 s6.13.9 - Address space qualifier functions. 2131 case Builtin::BIto_global: 2132 case Builtin::BIto_local: 2133 case Builtin::BIto_private: { 2134 auto Arg0 = EmitScalarExpr(E->getArg(0)); 2135 auto NewArgT = llvm::PointerType::get(Int8Ty, 2136 CGM.getContext().getTargetAddressSpace(LangAS::opencl_generic)); 2137 auto NewRetT = llvm::PointerType::get(Int8Ty, 2138 CGM.getContext().getTargetAddressSpace( 2139 E->getType()->getPointeeType().getAddressSpace())); 2140 auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT}, false); 2141 llvm::Value *NewArg; 2142 if (Arg0->getType()->getPointerAddressSpace() != 2143 NewArgT->getPointerAddressSpace()) 2144 NewArg = Builder.CreateAddrSpaceCast(Arg0, NewArgT); 2145 else 2146 NewArg = Builder.CreateBitOrPointerCast(Arg0, NewArgT); 2147 auto NewCall = Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, 2148 E->getDirectCallee()->getName()), {NewArg}); 2149 return RValue::get(Builder.CreateBitOrPointerCast(NewCall, 2150 ConvertType(E->getType()))); 2151 } 2152 2153 case Builtin::BIprintf: 2154 if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice) 2155 return EmitCUDADevicePrintfCallExpr(E, ReturnValue); 2156 break; 2157 case Builtin::BI__builtin_canonicalize: 2158 case Builtin::BI__builtin_canonicalizef: 2159 case Builtin::BI__builtin_canonicalizel: 2160 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::canonicalize)); 2161 } 2162 2163 // If this is an alias for a lib function (e.g. __builtin_sin), emit 2164 // the call using the normal call path, but using the unmangled 2165 // version of the function name. 2166 if (getContext().BuiltinInfo.isLibFunction(BuiltinID)) 2167 return emitLibraryCall(*this, FD, E, 2168 CGM.getBuiltinLibFunction(FD, BuiltinID)); 2169 2170 // If this is a predefined lib function (e.g. malloc), emit the call 2171 // using exactly the normal call path. 2172 if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID)) 2173 return emitLibraryCall(*this, FD, E, EmitScalarExpr(E->getCallee())); 2174 2175 // Check that a call to a target specific builtin has the correct target 2176 // features. 2177 // This is down here to avoid non-target specific builtins, however, if 2178 // generic builtins start to require generic target features then we 2179 // can move this up to the beginning of the function. 2180 checkTargetFeatures(E, FD); 2181 2182 // See if we have a target specific intrinsic. 2183 const char *Name = getContext().BuiltinInfo.getName(BuiltinID); 2184 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic; 2185 if (const char *Prefix = 2186 llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch())) { 2187 IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix, Name); 2188 // NOTE we dont need to perform a compatibility flag check here since the 2189 // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the 2190 // MS builtins via ALL_MS_LANGUAGES and are filtered earlier. 2191 if (IntrinsicID == Intrinsic::not_intrinsic) 2192 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix, Name); 2193 } 2194 2195 if (IntrinsicID != Intrinsic::not_intrinsic) { 2196 SmallVector<Value*, 16> Args; 2197 2198 // Find out if any arguments are required to be integer constant 2199 // expressions. 2200 unsigned ICEArguments = 0; 2201 ASTContext::GetBuiltinTypeError Error; 2202 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 2203 assert(Error == ASTContext::GE_None && "Should not codegen an error"); 2204 2205 Function *F = CGM.getIntrinsic(IntrinsicID); 2206 llvm::FunctionType *FTy = F->getFunctionType(); 2207 2208 for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) { 2209 Value *ArgValue; 2210 // If this is a normal argument, just emit it as a scalar. 2211 if ((ICEArguments & (1 << i)) == 0) { 2212 ArgValue = EmitScalarExpr(E->getArg(i)); 2213 } else { 2214 // If this is required to be a constant, constant fold it so that we 2215 // know that the generated intrinsic gets a ConstantInt. 2216 llvm::APSInt Result; 2217 bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result,getContext()); 2218 assert(IsConst && "Constant arg isn't actually constant?"); 2219 (void)IsConst; 2220 ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result); 2221 } 2222 2223 // If the intrinsic arg type is different from the builtin arg type 2224 // we need to do a bit cast. 2225 llvm::Type *PTy = FTy->getParamType(i); 2226 if (PTy != ArgValue->getType()) { 2227 assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) && 2228 "Must be able to losslessly bit cast to param"); 2229 ArgValue = Builder.CreateBitCast(ArgValue, PTy); 2230 } 2231 2232 Args.push_back(ArgValue); 2233 } 2234 2235 Value *V = Builder.CreateCall(F, Args); 2236 QualType BuiltinRetType = E->getType(); 2237 2238 llvm::Type *RetTy = VoidTy; 2239 if (!BuiltinRetType->isVoidType()) 2240 RetTy = ConvertType(BuiltinRetType); 2241 2242 if (RetTy != V->getType()) { 2243 assert(V->getType()->canLosslesslyBitCastTo(RetTy) && 2244 "Must be able to losslessly bit cast result type"); 2245 V = Builder.CreateBitCast(V, RetTy); 2246 } 2247 2248 return RValue::get(V); 2249 } 2250 2251 // See if we have a target specific builtin that needs to be lowered. 2252 if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E)) 2253 return RValue::get(V); 2254 2255 ErrorUnsupported(E, "builtin function"); 2256 2257 // Unknown builtin, for now just dump it out and return undef. 2258 return GetUndefRValue(E->getType()); 2259 } 2260 2261 static Value *EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, 2262 unsigned BuiltinID, const CallExpr *E, 2263 llvm::Triple::ArchType Arch) { 2264 switch (Arch) { 2265 case llvm::Triple::arm: 2266 case llvm::Triple::armeb: 2267 case llvm::Triple::thumb: 2268 case llvm::Triple::thumbeb: 2269 return CGF->EmitARMBuiltinExpr(BuiltinID, E); 2270 case llvm::Triple::aarch64: 2271 case llvm::Triple::aarch64_be: 2272 return CGF->EmitAArch64BuiltinExpr(BuiltinID, E); 2273 case llvm::Triple::x86: 2274 case llvm::Triple::x86_64: 2275 return CGF->EmitX86BuiltinExpr(BuiltinID, E); 2276 case llvm::Triple::ppc: 2277 case llvm::Triple::ppc64: 2278 case llvm::Triple::ppc64le: 2279 return CGF->EmitPPCBuiltinExpr(BuiltinID, E); 2280 case llvm::Triple::r600: 2281 case llvm::Triple::amdgcn: 2282 return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E); 2283 case llvm::Triple::systemz: 2284 return CGF->EmitSystemZBuiltinExpr(BuiltinID, E); 2285 case llvm::Triple::nvptx: 2286 case llvm::Triple::nvptx64: 2287 return CGF->EmitNVPTXBuiltinExpr(BuiltinID, E); 2288 case llvm::Triple::wasm32: 2289 case llvm::Triple::wasm64: 2290 return CGF->EmitWebAssemblyBuiltinExpr(BuiltinID, E); 2291 default: 2292 return nullptr; 2293 } 2294 } 2295 2296 Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID, 2297 const CallExpr *E) { 2298 if (getContext().BuiltinInfo.isAuxBuiltinID(BuiltinID)) { 2299 assert(getContext().getAuxTargetInfo() && "Missing aux target info"); 2300 return EmitTargetArchBuiltinExpr( 2301 this, getContext().BuiltinInfo.getAuxBuiltinID(BuiltinID), E, 2302 getContext().getAuxTargetInfo()->getTriple().getArch()); 2303 } 2304 2305 return EmitTargetArchBuiltinExpr(this, BuiltinID, E, 2306 getTarget().getTriple().getArch()); 2307 } 2308 2309 static llvm::VectorType *GetNeonType(CodeGenFunction *CGF, 2310 NeonTypeFlags TypeFlags, 2311 bool V1Ty=false) { 2312 int IsQuad = TypeFlags.isQuad(); 2313 switch (TypeFlags.getEltType()) { 2314 case NeonTypeFlags::Int8: 2315 case NeonTypeFlags::Poly8: 2316 return llvm::VectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad)); 2317 case NeonTypeFlags::Int16: 2318 case NeonTypeFlags::Poly16: 2319 case NeonTypeFlags::Float16: 2320 return llvm::VectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad)); 2321 case NeonTypeFlags::Int32: 2322 return llvm::VectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad)); 2323 case NeonTypeFlags::Int64: 2324 case NeonTypeFlags::Poly64: 2325 return llvm::VectorType::get(CGF->Int64Ty, V1Ty ? 1 : (1 << IsQuad)); 2326 case NeonTypeFlags::Poly128: 2327 // FIXME: i128 and f128 doesn't get fully support in Clang and llvm. 2328 // There is a lot of i128 and f128 API missing. 2329 // so we use v16i8 to represent poly128 and get pattern matched. 2330 return llvm::VectorType::get(CGF->Int8Ty, 16); 2331 case NeonTypeFlags::Float32: 2332 return llvm::VectorType::get(CGF->FloatTy, V1Ty ? 1 : (2 << IsQuad)); 2333 case NeonTypeFlags::Float64: 2334 return llvm::VectorType::get(CGF->DoubleTy, V1Ty ? 1 : (1 << IsQuad)); 2335 } 2336 llvm_unreachable("Unknown vector element type!"); 2337 } 2338 2339 static llvm::VectorType *GetFloatNeonType(CodeGenFunction *CGF, 2340 NeonTypeFlags IntTypeFlags) { 2341 int IsQuad = IntTypeFlags.isQuad(); 2342 switch (IntTypeFlags.getEltType()) { 2343 case NeonTypeFlags::Int32: 2344 return llvm::VectorType::get(CGF->FloatTy, (2 << IsQuad)); 2345 case NeonTypeFlags::Int64: 2346 return llvm::VectorType::get(CGF->DoubleTy, (1 << IsQuad)); 2347 default: 2348 llvm_unreachable("Type can't be converted to floating-point!"); 2349 } 2350 } 2351 2352 Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) { 2353 unsigned nElts = cast<llvm::VectorType>(V->getType())->getNumElements(); 2354 Value* SV = llvm::ConstantVector::getSplat(nElts, C); 2355 return Builder.CreateShuffleVector(V, V, SV, "lane"); 2356 } 2357 2358 Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops, 2359 const char *name, 2360 unsigned shift, bool rightshift) { 2361 unsigned j = 0; 2362 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end(); 2363 ai != ae; ++ai, ++j) 2364 if (shift > 0 && shift == j) 2365 Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift); 2366 else 2367 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name); 2368 2369 return Builder.CreateCall(F, Ops, name); 2370 } 2371 2372 Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty, 2373 bool neg) { 2374 int SV = cast<ConstantInt>(V)->getSExtValue(); 2375 return ConstantInt::get(Ty, neg ? -SV : SV); 2376 } 2377 2378 // \brief Right-shift a vector by a constant. 2379 Value *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift, 2380 llvm::Type *Ty, bool usgn, 2381 const char *name) { 2382 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); 2383 2384 int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue(); 2385 int EltSize = VTy->getScalarSizeInBits(); 2386 2387 Vec = Builder.CreateBitCast(Vec, Ty); 2388 2389 // lshr/ashr are undefined when the shift amount is equal to the vector 2390 // element size. 2391 if (ShiftAmt == EltSize) { 2392 if (usgn) { 2393 // Right-shifting an unsigned value by its size yields 0. 2394 return llvm::ConstantAggregateZero::get(VTy); 2395 } else { 2396 // Right-shifting a signed value by its size is equivalent 2397 // to a shift of size-1. 2398 --ShiftAmt; 2399 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt); 2400 } 2401 } 2402 2403 Shift = EmitNeonShiftVector(Shift, Ty, false); 2404 if (usgn) 2405 return Builder.CreateLShr(Vec, Shift, name); 2406 else 2407 return Builder.CreateAShr(Vec, Shift, name); 2408 } 2409 2410 enum { 2411 AddRetType = (1 << 0), 2412 Add1ArgType = (1 << 1), 2413 Add2ArgTypes = (1 << 2), 2414 2415 VectorizeRetType = (1 << 3), 2416 VectorizeArgTypes = (1 << 4), 2417 2418 InventFloatType = (1 << 5), 2419 UnsignedAlts = (1 << 6), 2420 2421 Use64BitVectors = (1 << 7), 2422 Use128BitVectors = (1 << 8), 2423 2424 Vectorize1ArgType = Add1ArgType | VectorizeArgTypes, 2425 VectorRet = AddRetType | VectorizeRetType, 2426 VectorRetGetArgs01 = 2427 AddRetType | Add2ArgTypes | VectorizeRetType | VectorizeArgTypes, 2428 FpCmpzModifiers = 2429 AddRetType | VectorizeRetType | Add1ArgType | InventFloatType 2430 }; 2431 2432 namespace { 2433 struct NeonIntrinsicInfo { 2434 const char *NameHint; 2435 unsigned BuiltinID; 2436 unsigned LLVMIntrinsic; 2437 unsigned AltLLVMIntrinsic; 2438 unsigned TypeModifier; 2439 2440 bool operator<(unsigned RHSBuiltinID) const { 2441 return BuiltinID < RHSBuiltinID; 2442 } 2443 bool operator<(const NeonIntrinsicInfo &TE) const { 2444 return BuiltinID < TE.BuiltinID; 2445 } 2446 }; 2447 } // end anonymous namespace 2448 2449 #define NEONMAP0(NameBase) \ 2450 { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 } 2451 2452 #define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \ 2453 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \ 2454 Intrinsic::LLVMIntrinsic, 0, TypeModifier } 2455 2456 #define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \ 2457 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \ 2458 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \ 2459 TypeModifier } 2460 2461 static const NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = { 2462 NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts), 2463 NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts), 2464 NEONMAP1(vabs_v, arm_neon_vabs, 0), 2465 NEONMAP1(vabsq_v, arm_neon_vabs, 0), 2466 NEONMAP0(vaddhn_v), 2467 NEONMAP1(vaesdq_v, arm_neon_aesd, 0), 2468 NEONMAP1(vaeseq_v, arm_neon_aese, 0), 2469 NEONMAP1(vaesimcq_v, arm_neon_aesimc, 0), 2470 NEONMAP1(vaesmcq_v, arm_neon_aesmc, 0), 2471 NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType), 2472 NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType), 2473 NEONMAP1(vcage_v, arm_neon_vacge, 0), 2474 NEONMAP1(vcageq_v, arm_neon_vacge, 0), 2475 NEONMAP1(vcagt_v, arm_neon_vacgt, 0), 2476 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0), 2477 NEONMAP1(vcale_v, arm_neon_vacge, 0), 2478 NEONMAP1(vcaleq_v, arm_neon_vacge, 0), 2479 NEONMAP1(vcalt_v, arm_neon_vacgt, 0), 2480 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0), 2481 NEONMAP1(vcls_v, arm_neon_vcls, Add1ArgType), 2482 NEONMAP1(vclsq_v, arm_neon_vcls, Add1ArgType), 2483 NEONMAP1(vclz_v, ctlz, Add1ArgType), 2484 NEONMAP1(vclzq_v, ctlz, Add1ArgType), 2485 NEONMAP1(vcnt_v, ctpop, Add1ArgType), 2486 NEONMAP1(vcntq_v, ctpop, Add1ArgType), 2487 NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0), 2488 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0), 2489 NEONMAP0(vcvt_f32_v), 2490 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), 2491 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0), 2492 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0), 2493 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0), 2494 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0), 2495 NEONMAP0(vcvt_s32_v), 2496 NEONMAP0(vcvt_s64_v), 2497 NEONMAP0(vcvt_u32_v), 2498 NEONMAP0(vcvt_u64_v), 2499 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0), 2500 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0), 2501 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0), 2502 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0), 2503 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0), 2504 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0), 2505 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0), 2506 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0), 2507 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0), 2508 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0), 2509 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0), 2510 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0), 2511 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0), 2512 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0), 2513 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0), 2514 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0), 2515 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0), 2516 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0), 2517 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0), 2518 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0), 2519 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0), 2520 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0), 2521 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0), 2522 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0), 2523 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0), 2524 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0), 2525 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0), 2526 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0), 2527 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0), 2528 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0), 2529 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0), 2530 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0), 2531 NEONMAP0(vcvtq_f32_v), 2532 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), 2533 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0), 2534 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0), 2535 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0), 2536 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0), 2537 NEONMAP0(vcvtq_s32_v), 2538 NEONMAP0(vcvtq_s64_v), 2539 NEONMAP0(vcvtq_u32_v), 2540 NEONMAP0(vcvtq_u64_v), 2541 NEONMAP0(vext_v), 2542 NEONMAP0(vextq_v), 2543 NEONMAP0(vfma_v), 2544 NEONMAP0(vfmaq_v), 2545 NEONMAP2(vhadd_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts), 2546 NEONMAP2(vhaddq_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts), 2547 NEONMAP2(vhsub_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts), 2548 NEONMAP2(vhsubq_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts), 2549 NEONMAP0(vld1_dup_v), 2550 NEONMAP1(vld1_v, arm_neon_vld1, 0), 2551 NEONMAP0(vld1q_dup_v), 2552 NEONMAP1(vld1q_v, arm_neon_vld1, 0), 2553 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0), 2554 NEONMAP1(vld2_v, arm_neon_vld2, 0), 2555 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0), 2556 NEONMAP1(vld2q_v, arm_neon_vld2, 0), 2557 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0), 2558 NEONMAP1(vld3_v, arm_neon_vld3, 0), 2559 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0), 2560 NEONMAP1(vld3q_v, arm_neon_vld3, 0), 2561 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0), 2562 NEONMAP1(vld4_v, arm_neon_vld4, 0), 2563 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0), 2564 NEONMAP1(vld4q_v, arm_neon_vld4, 0), 2565 NEONMAP2(vmax_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts), 2566 NEONMAP1(vmaxnm_v, arm_neon_vmaxnm, Add1ArgType), 2567 NEONMAP1(vmaxnmq_v, arm_neon_vmaxnm, Add1ArgType), 2568 NEONMAP2(vmaxq_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts), 2569 NEONMAP2(vmin_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts), 2570 NEONMAP1(vminnm_v, arm_neon_vminnm, Add1ArgType), 2571 NEONMAP1(vminnmq_v, arm_neon_vminnm, Add1ArgType), 2572 NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts), 2573 NEONMAP0(vmovl_v), 2574 NEONMAP0(vmovn_v), 2575 NEONMAP1(vmul_v, arm_neon_vmulp, Add1ArgType), 2576 NEONMAP0(vmull_v), 2577 NEONMAP1(vmulq_v, arm_neon_vmulp, Add1ArgType), 2578 NEONMAP2(vpadal_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts), 2579 NEONMAP2(vpadalq_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts), 2580 NEONMAP1(vpadd_v, arm_neon_vpadd, Add1ArgType), 2581 NEONMAP2(vpaddl_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts), 2582 NEONMAP2(vpaddlq_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts), 2583 NEONMAP1(vpaddq_v, arm_neon_vpadd, Add1ArgType), 2584 NEONMAP2(vpmax_v, arm_neon_vpmaxu, arm_neon_vpmaxs, Add1ArgType | UnsignedAlts), 2585 NEONMAP2(vpmin_v, arm_neon_vpminu, arm_neon_vpmins, Add1ArgType | UnsignedAlts), 2586 NEONMAP1(vqabs_v, arm_neon_vqabs, Add1ArgType), 2587 NEONMAP1(vqabsq_v, arm_neon_vqabs, Add1ArgType), 2588 NEONMAP2(vqadd_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts), 2589 NEONMAP2(vqaddq_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts), 2590 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, arm_neon_vqadds, 0), 2591 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, arm_neon_vqsubs, 0), 2592 NEONMAP1(vqdmulh_v, arm_neon_vqdmulh, Add1ArgType), 2593 NEONMAP1(vqdmulhq_v, arm_neon_vqdmulh, Add1ArgType), 2594 NEONMAP1(vqdmull_v, arm_neon_vqdmull, Add1ArgType), 2595 NEONMAP2(vqmovn_v, arm_neon_vqmovnu, arm_neon_vqmovns, Add1ArgType | UnsignedAlts), 2596 NEONMAP1(vqmovun_v, arm_neon_vqmovnsu, Add1ArgType), 2597 NEONMAP1(vqneg_v, arm_neon_vqneg, Add1ArgType), 2598 NEONMAP1(vqnegq_v, arm_neon_vqneg, Add1ArgType), 2599 NEONMAP1(vqrdmulh_v, arm_neon_vqrdmulh, Add1ArgType), 2600 NEONMAP1(vqrdmulhq_v, arm_neon_vqrdmulh, Add1ArgType), 2601 NEONMAP2(vqrshl_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts), 2602 NEONMAP2(vqrshlq_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts), 2603 NEONMAP2(vqshl_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts), 2604 NEONMAP2(vqshl_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts), 2605 NEONMAP2(vqshlq_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts), 2606 NEONMAP2(vqshlq_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts), 2607 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0), 2608 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0), 2609 NEONMAP2(vqsub_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts), 2610 NEONMAP2(vqsubq_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts), 2611 NEONMAP1(vraddhn_v, arm_neon_vraddhn, Add1ArgType), 2612 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0), 2613 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0), 2614 NEONMAP1(vrecps_v, arm_neon_vrecps, Add1ArgType), 2615 NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType), 2616 NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts), 2617 NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts), 2618 NEONMAP1(vrnd_v, arm_neon_vrintz, Add1ArgType), 2619 NEONMAP1(vrnda_v, arm_neon_vrinta, Add1ArgType), 2620 NEONMAP1(vrndaq_v, arm_neon_vrinta, Add1ArgType), 2621 NEONMAP1(vrndm_v, arm_neon_vrintm, Add1ArgType), 2622 NEONMAP1(vrndmq_v, arm_neon_vrintm, Add1ArgType), 2623 NEONMAP1(vrndn_v, arm_neon_vrintn, Add1ArgType), 2624 NEONMAP1(vrndnq_v, arm_neon_vrintn, Add1ArgType), 2625 NEONMAP1(vrndp_v, arm_neon_vrintp, Add1ArgType), 2626 NEONMAP1(vrndpq_v, arm_neon_vrintp, Add1ArgType), 2627 NEONMAP1(vrndq_v, arm_neon_vrintz, Add1ArgType), 2628 NEONMAP1(vrndx_v, arm_neon_vrintx, Add1ArgType), 2629 NEONMAP1(vrndxq_v, arm_neon_vrintx, Add1ArgType), 2630 NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts), 2631 NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts), 2632 NEONMAP2(vrshr_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts), 2633 NEONMAP2(vrshrq_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts), 2634 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0), 2635 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0), 2636 NEONMAP1(vrsqrts_v, arm_neon_vrsqrts, Add1ArgType), 2637 NEONMAP1(vrsqrtsq_v, arm_neon_vrsqrts, Add1ArgType), 2638 NEONMAP1(vrsubhn_v, arm_neon_vrsubhn, Add1ArgType), 2639 NEONMAP1(vsha1su0q_v, arm_neon_sha1su0, 0), 2640 NEONMAP1(vsha1su1q_v, arm_neon_sha1su1, 0), 2641 NEONMAP1(vsha256h2q_v, arm_neon_sha256h2, 0), 2642 NEONMAP1(vsha256hq_v, arm_neon_sha256h, 0), 2643 NEONMAP1(vsha256su0q_v, arm_neon_sha256su0, 0), 2644 NEONMAP1(vsha256su1q_v, arm_neon_sha256su1, 0), 2645 NEONMAP0(vshl_n_v), 2646 NEONMAP2(vshl_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts), 2647 NEONMAP0(vshll_n_v), 2648 NEONMAP0(vshlq_n_v), 2649 NEONMAP2(vshlq_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts), 2650 NEONMAP0(vshr_n_v), 2651 NEONMAP0(vshrn_n_v), 2652 NEONMAP0(vshrq_n_v), 2653 NEONMAP1(vst1_v, arm_neon_vst1, 0), 2654 NEONMAP1(vst1q_v, arm_neon_vst1, 0), 2655 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0), 2656 NEONMAP1(vst2_v, arm_neon_vst2, 0), 2657 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0), 2658 NEONMAP1(vst2q_v, arm_neon_vst2, 0), 2659 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0), 2660 NEONMAP1(vst3_v, arm_neon_vst3, 0), 2661 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0), 2662 NEONMAP1(vst3q_v, arm_neon_vst3, 0), 2663 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0), 2664 NEONMAP1(vst4_v, arm_neon_vst4, 0), 2665 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0), 2666 NEONMAP1(vst4q_v, arm_neon_vst4, 0), 2667 NEONMAP0(vsubhn_v), 2668 NEONMAP0(vtrn_v), 2669 NEONMAP0(vtrnq_v), 2670 NEONMAP0(vtst_v), 2671 NEONMAP0(vtstq_v), 2672 NEONMAP0(vuzp_v), 2673 NEONMAP0(vuzpq_v), 2674 NEONMAP0(vzip_v), 2675 NEONMAP0(vzipq_v) 2676 }; 2677 2678 static const NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = { 2679 NEONMAP1(vabs_v, aarch64_neon_abs, 0), 2680 NEONMAP1(vabsq_v, aarch64_neon_abs, 0), 2681 NEONMAP0(vaddhn_v), 2682 NEONMAP1(vaesdq_v, aarch64_crypto_aesd, 0), 2683 NEONMAP1(vaeseq_v, aarch64_crypto_aese, 0), 2684 NEONMAP1(vaesimcq_v, aarch64_crypto_aesimc, 0), 2685 NEONMAP1(vaesmcq_v, aarch64_crypto_aesmc, 0), 2686 NEONMAP1(vcage_v, aarch64_neon_facge, 0), 2687 NEONMAP1(vcageq_v, aarch64_neon_facge, 0), 2688 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0), 2689 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0), 2690 NEONMAP1(vcale_v, aarch64_neon_facge, 0), 2691 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0), 2692 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0), 2693 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0), 2694 NEONMAP1(vcls_v, aarch64_neon_cls, Add1ArgType), 2695 NEONMAP1(vclsq_v, aarch64_neon_cls, Add1ArgType), 2696 NEONMAP1(vclz_v, ctlz, Add1ArgType), 2697 NEONMAP1(vclzq_v, ctlz, Add1ArgType), 2698 NEONMAP1(vcnt_v, ctpop, Add1ArgType), 2699 NEONMAP1(vcntq_v, ctpop, Add1ArgType), 2700 NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0), 2701 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0), 2702 NEONMAP0(vcvt_f32_v), 2703 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 2704 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 2705 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0), 2706 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0), 2707 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0), 2708 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0), 2709 NEONMAP0(vcvtq_f32_v), 2710 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 2711 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 2712 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0), 2713 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0), 2714 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0), 2715 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0), 2716 NEONMAP1(vcvtx_f32_v, aarch64_neon_fcvtxn, AddRetType | Add1ArgType), 2717 NEONMAP0(vext_v), 2718 NEONMAP0(vextq_v), 2719 NEONMAP0(vfma_v), 2720 NEONMAP0(vfmaq_v), 2721 NEONMAP2(vhadd_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts), 2722 NEONMAP2(vhaddq_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts), 2723 NEONMAP2(vhsub_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts), 2724 NEONMAP2(vhsubq_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts), 2725 NEONMAP0(vmovl_v), 2726 NEONMAP0(vmovn_v), 2727 NEONMAP1(vmul_v, aarch64_neon_pmul, Add1ArgType), 2728 NEONMAP1(vmulq_v, aarch64_neon_pmul, Add1ArgType), 2729 NEONMAP1(vpadd_v, aarch64_neon_addp, Add1ArgType), 2730 NEONMAP2(vpaddl_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts), 2731 NEONMAP2(vpaddlq_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts), 2732 NEONMAP1(vpaddq_v, aarch64_neon_addp, Add1ArgType), 2733 NEONMAP1(vqabs_v, aarch64_neon_sqabs, Add1ArgType), 2734 NEONMAP1(vqabsq_v, aarch64_neon_sqabs, Add1ArgType), 2735 NEONMAP2(vqadd_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts), 2736 NEONMAP2(vqaddq_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts), 2737 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0), 2738 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0), 2739 NEONMAP1(vqdmulh_v, aarch64_neon_sqdmulh, Add1ArgType), 2740 NEONMAP1(vqdmulhq_v, aarch64_neon_sqdmulh, Add1ArgType), 2741 NEONMAP1(vqdmull_v, aarch64_neon_sqdmull, Add1ArgType), 2742 NEONMAP2(vqmovn_v, aarch64_neon_uqxtn, aarch64_neon_sqxtn, Add1ArgType | UnsignedAlts), 2743 NEONMAP1(vqmovun_v, aarch64_neon_sqxtun, Add1ArgType), 2744 NEONMAP1(vqneg_v, aarch64_neon_sqneg, Add1ArgType), 2745 NEONMAP1(vqnegq_v, aarch64_neon_sqneg, Add1ArgType), 2746 NEONMAP1(vqrdmulh_v, aarch64_neon_sqrdmulh, Add1ArgType), 2747 NEONMAP1(vqrdmulhq_v, aarch64_neon_sqrdmulh, Add1ArgType), 2748 NEONMAP2(vqrshl_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts), 2749 NEONMAP2(vqrshlq_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts), 2750 NEONMAP2(vqshl_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl, UnsignedAlts), 2751 NEONMAP2(vqshl_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts), 2752 NEONMAP2(vqshlq_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl,UnsignedAlts), 2753 NEONMAP2(vqshlq_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts), 2754 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0), 2755 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0), 2756 NEONMAP2(vqsub_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts), 2757 NEONMAP2(vqsubq_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts), 2758 NEONMAP1(vraddhn_v, aarch64_neon_raddhn, Add1ArgType), 2759 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0), 2760 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0), 2761 NEONMAP1(vrecps_v, aarch64_neon_frecps, Add1ArgType), 2762 NEONMAP1(vrecpsq_v, aarch64_neon_frecps, Add1ArgType), 2763 NEONMAP2(vrhadd_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts), 2764 NEONMAP2(vrhaddq_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts), 2765 NEONMAP2(vrshl_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts), 2766 NEONMAP2(vrshlq_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts), 2767 NEONMAP2(vrshr_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts), 2768 NEONMAP2(vrshrq_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts), 2769 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0), 2770 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0), 2771 NEONMAP1(vrsqrts_v, aarch64_neon_frsqrts, Add1ArgType), 2772 NEONMAP1(vrsqrtsq_v, aarch64_neon_frsqrts, Add1ArgType), 2773 NEONMAP1(vrsubhn_v, aarch64_neon_rsubhn, Add1ArgType), 2774 NEONMAP1(vsha1su0q_v, aarch64_crypto_sha1su0, 0), 2775 NEONMAP1(vsha1su1q_v, aarch64_crypto_sha1su1, 0), 2776 NEONMAP1(vsha256h2q_v, aarch64_crypto_sha256h2, 0), 2777 NEONMAP1(vsha256hq_v, aarch64_crypto_sha256h, 0), 2778 NEONMAP1(vsha256su0q_v, aarch64_crypto_sha256su0, 0), 2779 NEONMAP1(vsha256su1q_v, aarch64_crypto_sha256su1, 0), 2780 NEONMAP0(vshl_n_v), 2781 NEONMAP2(vshl_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts), 2782 NEONMAP0(vshll_n_v), 2783 NEONMAP0(vshlq_n_v), 2784 NEONMAP2(vshlq_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts), 2785 NEONMAP0(vshr_n_v), 2786 NEONMAP0(vshrn_n_v), 2787 NEONMAP0(vshrq_n_v), 2788 NEONMAP0(vsubhn_v), 2789 NEONMAP0(vtst_v), 2790 NEONMAP0(vtstq_v), 2791 }; 2792 2793 static const NeonIntrinsicInfo AArch64SISDIntrinsicMap[] = { 2794 NEONMAP1(vabdd_f64, aarch64_sisd_fabd, Add1ArgType), 2795 NEONMAP1(vabds_f32, aarch64_sisd_fabd, Add1ArgType), 2796 NEONMAP1(vabsd_s64, aarch64_neon_abs, Add1ArgType), 2797 NEONMAP1(vaddlv_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType), 2798 NEONMAP1(vaddlv_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType), 2799 NEONMAP1(vaddlvq_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType), 2800 NEONMAP1(vaddlvq_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType), 2801 NEONMAP1(vaddv_f32, aarch64_neon_faddv, AddRetType | Add1ArgType), 2802 NEONMAP1(vaddv_s32, aarch64_neon_saddv, AddRetType | Add1ArgType), 2803 NEONMAP1(vaddv_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType), 2804 NEONMAP1(vaddvq_f32, aarch64_neon_faddv, AddRetType | Add1ArgType), 2805 NEONMAP1(vaddvq_f64, aarch64_neon_faddv, AddRetType | Add1ArgType), 2806 NEONMAP1(vaddvq_s32, aarch64_neon_saddv, AddRetType | Add1ArgType), 2807 NEONMAP1(vaddvq_s64, aarch64_neon_saddv, AddRetType | Add1ArgType), 2808 NEONMAP1(vaddvq_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType), 2809 NEONMAP1(vaddvq_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 2810 NEONMAP1(vcaged_f64, aarch64_neon_facge, AddRetType | Add1ArgType), 2811 NEONMAP1(vcages_f32, aarch64_neon_facge, AddRetType | Add1ArgType), 2812 NEONMAP1(vcagtd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType), 2813 NEONMAP1(vcagts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType), 2814 NEONMAP1(vcaled_f64, aarch64_neon_facge, AddRetType | Add1ArgType), 2815 NEONMAP1(vcales_f32, aarch64_neon_facge, AddRetType | Add1ArgType), 2816 NEONMAP1(vcaltd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType), 2817 NEONMAP1(vcalts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType), 2818 NEONMAP1(vcvtad_s64_f64, aarch64_neon_fcvtas, AddRetType | Add1ArgType), 2819 NEONMAP1(vcvtad_u64_f64, aarch64_neon_fcvtau, AddRetType | Add1ArgType), 2820 NEONMAP1(vcvtas_s32_f32, aarch64_neon_fcvtas, AddRetType | Add1ArgType), 2821 NEONMAP1(vcvtas_u32_f32, aarch64_neon_fcvtau, AddRetType | Add1ArgType), 2822 NEONMAP1(vcvtd_n_f64_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType), 2823 NEONMAP1(vcvtd_n_f64_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType), 2824 NEONMAP1(vcvtd_n_s64_f64, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType), 2825 NEONMAP1(vcvtd_n_u64_f64, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType), 2826 NEONMAP1(vcvtmd_s64_f64, aarch64_neon_fcvtms, AddRetType | Add1ArgType), 2827 NEONMAP1(vcvtmd_u64_f64, aarch64_neon_fcvtmu, AddRetType | Add1ArgType), 2828 NEONMAP1(vcvtms_s32_f32, aarch64_neon_fcvtms, AddRetType | Add1ArgType), 2829 NEONMAP1(vcvtms_u32_f32, aarch64_neon_fcvtmu, AddRetType | Add1ArgType), 2830 NEONMAP1(vcvtnd_s64_f64, aarch64_neon_fcvtns, AddRetType | Add1ArgType), 2831 NEONMAP1(vcvtnd_u64_f64, aarch64_neon_fcvtnu, AddRetType | Add1ArgType), 2832 NEONMAP1(vcvtns_s32_f32, aarch64_neon_fcvtns, AddRetType | Add1ArgType), 2833 NEONMAP1(vcvtns_u32_f32, aarch64_neon_fcvtnu, AddRetType | Add1ArgType), 2834 NEONMAP1(vcvtpd_s64_f64, aarch64_neon_fcvtps, AddRetType | Add1ArgType), 2835 NEONMAP1(vcvtpd_u64_f64, aarch64_neon_fcvtpu, AddRetType | Add1ArgType), 2836 NEONMAP1(vcvtps_s32_f32, aarch64_neon_fcvtps, AddRetType | Add1ArgType), 2837 NEONMAP1(vcvtps_u32_f32, aarch64_neon_fcvtpu, AddRetType | Add1ArgType), 2838 NEONMAP1(vcvts_n_f32_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType), 2839 NEONMAP1(vcvts_n_f32_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType), 2840 NEONMAP1(vcvts_n_s32_f32, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType), 2841 NEONMAP1(vcvts_n_u32_f32, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType), 2842 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0), 2843 NEONMAP1(vmaxnmv_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 2844 NEONMAP1(vmaxnmvq_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 2845 NEONMAP1(vmaxnmvq_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 2846 NEONMAP1(vmaxv_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 2847 NEONMAP1(vmaxv_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType), 2848 NEONMAP1(vmaxv_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType), 2849 NEONMAP1(vmaxvq_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 2850 NEONMAP1(vmaxvq_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 2851 NEONMAP1(vmaxvq_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType), 2852 NEONMAP1(vmaxvq_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType), 2853 NEONMAP1(vminnmv_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 2854 NEONMAP1(vminnmvq_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 2855 NEONMAP1(vminnmvq_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 2856 NEONMAP1(vminv_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 2857 NEONMAP1(vminv_s32, aarch64_neon_sminv, AddRetType | Add1ArgType), 2858 NEONMAP1(vminv_u32, aarch64_neon_uminv, AddRetType | Add1ArgType), 2859 NEONMAP1(vminvq_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 2860 NEONMAP1(vminvq_f64, aarch64_neon_fminv, AddRetType | Add1ArgType), 2861 NEONMAP1(vminvq_s32, aarch64_neon_sminv, AddRetType | Add1ArgType), 2862 NEONMAP1(vminvq_u32, aarch64_neon_uminv, AddRetType | Add1ArgType), 2863 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0), 2864 NEONMAP1(vmulxd_f64, aarch64_neon_fmulx, Add1ArgType), 2865 NEONMAP1(vmulxs_f32, aarch64_neon_fmulx, Add1ArgType), 2866 NEONMAP1(vpaddd_s64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 2867 NEONMAP1(vpaddd_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 2868 NEONMAP1(vpmaxnmqd_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 2869 NEONMAP1(vpmaxnms_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 2870 NEONMAP1(vpmaxqd_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 2871 NEONMAP1(vpmaxs_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 2872 NEONMAP1(vpminnmqd_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 2873 NEONMAP1(vpminnms_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 2874 NEONMAP1(vpminqd_f64, aarch64_neon_fminv, AddRetType | Add1ArgType), 2875 NEONMAP1(vpmins_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 2876 NEONMAP1(vqabsb_s8, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors), 2877 NEONMAP1(vqabsd_s64, aarch64_neon_sqabs, Add1ArgType), 2878 NEONMAP1(vqabsh_s16, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors), 2879 NEONMAP1(vqabss_s32, aarch64_neon_sqabs, Add1ArgType), 2880 NEONMAP1(vqaddb_s8, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors), 2881 NEONMAP1(vqaddb_u8, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors), 2882 NEONMAP1(vqaddd_s64, aarch64_neon_sqadd, Add1ArgType), 2883 NEONMAP1(vqaddd_u64, aarch64_neon_uqadd, Add1ArgType), 2884 NEONMAP1(vqaddh_s16, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors), 2885 NEONMAP1(vqaddh_u16, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors), 2886 NEONMAP1(vqadds_s32, aarch64_neon_sqadd, Add1ArgType), 2887 NEONMAP1(vqadds_u32, aarch64_neon_uqadd, Add1ArgType), 2888 NEONMAP1(vqdmulhh_s16, aarch64_neon_sqdmulh, Vectorize1ArgType | Use64BitVectors), 2889 NEONMAP1(vqdmulhs_s32, aarch64_neon_sqdmulh, Add1ArgType), 2890 NEONMAP1(vqdmullh_s16, aarch64_neon_sqdmull, VectorRet | Use128BitVectors), 2891 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0), 2892 NEONMAP1(vqmovnd_s64, aarch64_neon_scalar_sqxtn, AddRetType | Add1ArgType), 2893 NEONMAP1(vqmovnd_u64, aarch64_neon_scalar_uqxtn, AddRetType | Add1ArgType), 2894 NEONMAP1(vqmovnh_s16, aarch64_neon_sqxtn, VectorRet | Use64BitVectors), 2895 NEONMAP1(vqmovnh_u16, aarch64_neon_uqxtn, VectorRet | Use64BitVectors), 2896 NEONMAP1(vqmovns_s32, aarch64_neon_sqxtn, VectorRet | Use64BitVectors), 2897 NEONMAP1(vqmovns_u32, aarch64_neon_uqxtn, VectorRet | Use64BitVectors), 2898 NEONMAP1(vqmovund_s64, aarch64_neon_scalar_sqxtun, AddRetType | Add1ArgType), 2899 NEONMAP1(vqmovunh_s16, aarch64_neon_sqxtun, VectorRet | Use64BitVectors), 2900 NEONMAP1(vqmovuns_s32, aarch64_neon_sqxtun, VectorRet | Use64BitVectors), 2901 NEONMAP1(vqnegb_s8, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors), 2902 NEONMAP1(vqnegd_s64, aarch64_neon_sqneg, Add1ArgType), 2903 NEONMAP1(vqnegh_s16, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors), 2904 NEONMAP1(vqnegs_s32, aarch64_neon_sqneg, Add1ArgType), 2905 NEONMAP1(vqrdmulhh_s16, aarch64_neon_sqrdmulh, Vectorize1ArgType | Use64BitVectors), 2906 NEONMAP1(vqrdmulhs_s32, aarch64_neon_sqrdmulh, Add1ArgType), 2907 NEONMAP1(vqrshlb_s8, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors), 2908 NEONMAP1(vqrshlb_u8, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors), 2909 NEONMAP1(vqrshld_s64, aarch64_neon_sqrshl, Add1ArgType), 2910 NEONMAP1(vqrshld_u64, aarch64_neon_uqrshl, Add1ArgType), 2911 NEONMAP1(vqrshlh_s16, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors), 2912 NEONMAP1(vqrshlh_u16, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors), 2913 NEONMAP1(vqrshls_s32, aarch64_neon_sqrshl, Add1ArgType), 2914 NEONMAP1(vqrshls_u32, aarch64_neon_uqrshl, Add1ArgType), 2915 NEONMAP1(vqrshrnd_n_s64, aarch64_neon_sqrshrn, AddRetType), 2916 NEONMAP1(vqrshrnd_n_u64, aarch64_neon_uqrshrn, AddRetType), 2917 NEONMAP1(vqrshrnh_n_s16, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors), 2918 NEONMAP1(vqrshrnh_n_u16, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors), 2919 NEONMAP1(vqrshrns_n_s32, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors), 2920 NEONMAP1(vqrshrns_n_u32, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors), 2921 NEONMAP1(vqrshrund_n_s64, aarch64_neon_sqrshrun, AddRetType), 2922 NEONMAP1(vqrshrunh_n_s16, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors), 2923 NEONMAP1(vqrshruns_n_s32, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors), 2924 NEONMAP1(vqshlb_n_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 2925 NEONMAP1(vqshlb_n_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 2926 NEONMAP1(vqshlb_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 2927 NEONMAP1(vqshlb_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 2928 NEONMAP1(vqshld_s64, aarch64_neon_sqshl, Add1ArgType), 2929 NEONMAP1(vqshld_u64, aarch64_neon_uqshl, Add1ArgType), 2930 NEONMAP1(vqshlh_n_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 2931 NEONMAP1(vqshlh_n_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 2932 NEONMAP1(vqshlh_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 2933 NEONMAP1(vqshlh_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 2934 NEONMAP1(vqshls_n_s32, aarch64_neon_sqshl, Add1ArgType), 2935 NEONMAP1(vqshls_n_u32, aarch64_neon_uqshl, Add1ArgType), 2936 NEONMAP1(vqshls_s32, aarch64_neon_sqshl, Add1ArgType), 2937 NEONMAP1(vqshls_u32, aarch64_neon_uqshl, Add1ArgType), 2938 NEONMAP1(vqshlub_n_s8, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors), 2939 NEONMAP1(vqshluh_n_s16, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors), 2940 NEONMAP1(vqshlus_n_s32, aarch64_neon_sqshlu, Add1ArgType), 2941 NEONMAP1(vqshrnd_n_s64, aarch64_neon_sqshrn, AddRetType), 2942 NEONMAP1(vqshrnd_n_u64, aarch64_neon_uqshrn, AddRetType), 2943 NEONMAP1(vqshrnh_n_s16, aarch64_neon_sqshrn, VectorRet | Use64BitVectors), 2944 NEONMAP1(vqshrnh_n_u16, aarch64_neon_uqshrn, VectorRet | Use64BitVectors), 2945 NEONMAP1(vqshrns_n_s32, aarch64_neon_sqshrn, VectorRet | Use64BitVectors), 2946 NEONMAP1(vqshrns_n_u32, aarch64_neon_uqshrn, VectorRet | Use64BitVectors), 2947 NEONMAP1(vqshrund_n_s64, aarch64_neon_sqshrun, AddRetType), 2948 NEONMAP1(vqshrunh_n_s16, aarch64_neon_sqshrun, VectorRet | Use64BitVectors), 2949 NEONMAP1(vqshruns_n_s32, aarch64_neon_sqshrun, VectorRet | Use64BitVectors), 2950 NEONMAP1(vqsubb_s8, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors), 2951 NEONMAP1(vqsubb_u8, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors), 2952 NEONMAP1(vqsubd_s64, aarch64_neon_sqsub, Add1ArgType), 2953 NEONMAP1(vqsubd_u64, aarch64_neon_uqsub, Add1ArgType), 2954 NEONMAP1(vqsubh_s16, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors), 2955 NEONMAP1(vqsubh_u16, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors), 2956 NEONMAP1(vqsubs_s32, aarch64_neon_sqsub, Add1ArgType), 2957 NEONMAP1(vqsubs_u32, aarch64_neon_uqsub, Add1ArgType), 2958 NEONMAP1(vrecped_f64, aarch64_neon_frecpe, Add1ArgType), 2959 NEONMAP1(vrecpes_f32, aarch64_neon_frecpe, Add1ArgType), 2960 NEONMAP1(vrecpxd_f64, aarch64_neon_frecpx, Add1ArgType), 2961 NEONMAP1(vrecpxs_f32, aarch64_neon_frecpx, Add1ArgType), 2962 NEONMAP1(vrshld_s64, aarch64_neon_srshl, Add1ArgType), 2963 NEONMAP1(vrshld_u64, aarch64_neon_urshl, Add1ArgType), 2964 NEONMAP1(vrsqrted_f64, aarch64_neon_frsqrte, Add1ArgType), 2965 NEONMAP1(vrsqrtes_f32, aarch64_neon_frsqrte, Add1ArgType), 2966 NEONMAP1(vrsqrtsd_f64, aarch64_neon_frsqrts, Add1ArgType), 2967 NEONMAP1(vrsqrtss_f32, aarch64_neon_frsqrts, Add1ArgType), 2968 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0), 2969 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0), 2970 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0), 2971 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0), 2972 NEONMAP1(vshld_s64, aarch64_neon_sshl, Add1ArgType), 2973 NEONMAP1(vshld_u64, aarch64_neon_ushl, Add1ArgType), 2974 NEONMAP1(vslid_n_s64, aarch64_neon_vsli, Vectorize1ArgType), 2975 NEONMAP1(vslid_n_u64, aarch64_neon_vsli, Vectorize1ArgType), 2976 NEONMAP1(vsqaddb_u8, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors), 2977 NEONMAP1(vsqaddd_u64, aarch64_neon_usqadd, Add1ArgType), 2978 NEONMAP1(vsqaddh_u16, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors), 2979 NEONMAP1(vsqadds_u32, aarch64_neon_usqadd, Add1ArgType), 2980 NEONMAP1(vsrid_n_s64, aarch64_neon_vsri, Vectorize1ArgType), 2981 NEONMAP1(vsrid_n_u64, aarch64_neon_vsri, Vectorize1ArgType), 2982 NEONMAP1(vuqaddb_s8, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors), 2983 NEONMAP1(vuqaddd_s64, aarch64_neon_suqadd, Add1ArgType), 2984 NEONMAP1(vuqaddh_s16, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors), 2985 NEONMAP1(vuqadds_s32, aarch64_neon_suqadd, Add1ArgType), 2986 }; 2987 2988 #undef NEONMAP0 2989 #undef NEONMAP1 2990 #undef NEONMAP2 2991 2992 static bool NEONSIMDIntrinsicsProvenSorted = false; 2993 2994 static bool AArch64SIMDIntrinsicsProvenSorted = false; 2995 static bool AArch64SISDIntrinsicsProvenSorted = false; 2996 2997 2998 static const NeonIntrinsicInfo * 2999 findNeonIntrinsicInMap(ArrayRef<NeonIntrinsicInfo> IntrinsicMap, 3000 unsigned BuiltinID, bool &MapProvenSorted) { 3001 3002 #ifndef NDEBUG 3003 if (!MapProvenSorted) { 3004 assert(std::is_sorted(std::begin(IntrinsicMap), std::end(IntrinsicMap))); 3005 MapProvenSorted = true; 3006 } 3007 #endif 3008 3009 const NeonIntrinsicInfo *Builtin = 3010 std::lower_bound(IntrinsicMap.begin(), IntrinsicMap.end(), BuiltinID); 3011 3012 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID) 3013 return Builtin; 3014 3015 return nullptr; 3016 } 3017 3018 Function *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID, 3019 unsigned Modifier, 3020 llvm::Type *ArgType, 3021 const CallExpr *E) { 3022 int VectorSize = 0; 3023 if (Modifier & Use64BitVectors) 3024 VectorSize = 64; 3025 else if (Modifier & Use128BitVectors) 3026 VectorSize = 128; 3027 3028 // Return type. 3029 SmallVector<llvm::Type *, 3> Tys; 3030 if (Modifier & AddRetType) { 3031 llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext())); 3032 if (Modifier & VectorizeRetType) 3033 Ty = llvm::VectorType::get( 3034 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1); 3035 3036 Tys.push_back(Ty); 3037 } 3038 3039 // Arguments. 3040 if (Modifier & VectorizeArgTypes) { 3041 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1; 3042 ArgType = llvm::VectorType::get(ArgType, Elts); 3043 } 3044 3045 if (Modifier & (Add1ArgType | Add2ArgTypes)) 3046 Tys.push_back(ArgType); 3047 3048 if (Modifier & Add2ArgTypes) 3049 Tys.push_back(ArgType); 3050 3051 if (Modifier & InventFloatType) 3052 Tys.push_back(FloatTy); 3053 3054 return CGM.getIntrinsic(IntrinsicID, Tys); 3055 } 3056 3057 static Value *EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, 3058 const NeonIntrinsicInfo &SISDInfo, 3059 SmallVectorImpl<Value *> &Ops, 3060 const CallExpr *E) { 3061 unsigned BuiltinID = SISDInfo.BuiltinID; 3062 unsigned int Int = SISDInfo.LLVMIntrinsic; 3063 unsigned Modifier = SISDInfo.TypeModifier; 3064 const char *s = SISDInfo.NameHint; 3065 3066 switch (BuiltinID) { 3067 case NEON::BI__builtin_neon_vcled_s64: 3068 case NEON::BI__builtin_neon_vcled_u64: 3069 case NEON::BI__builtin_neon_vcles_f32: 3070 case NEON::BI__builtin_neon_vcled_f64: 3071 case NEON::BI__builtin_neon_vcltd_s64: 3072 case NEON::BI__builtin_neon_vcltd_u64: 3073 case NEON::BI__builtin_neon_vclts_f32: 3074 case NEON::BI__builtin_neon_vcltd_f64: 3075 case NEON::BI__builtin_neon_vcales_f32: 3076 case NEON::BI__builtin_neon_vcaled_f64: 3077 case NEON::BI__builtin_neon_vcalts_f32: 3078 case NEON::BI__builtin_neon_vcaltd_f64: 3079 // Only one direction of comparisons actually exist, cmle is actually a cmge 3080 // with swapped operands. The table gives us the right intrinsic but we 3081 // still need to do the swap. 3082 std::swap(Ops[0], Ops[1]); 3083 break; 3084 } 3085 3086 assert(Int && "Generic code assumes a valid intrinsic"); 3087 3088 // Determine the type(s) of this overloaded AArch64 intrinsic. 3089 const Expr *Arg = E->getArg(0); 3090 llvm::Type *ArgTy = CGF.ConvertType(Arg->getType()); 3091 Function *F = CGF.LookupNeonLLVMIntrinsic(Int, Modifier, ArgTy, E); 3092 3093 int j = 0; 3094 ConstantInt *C0 = ConstantInt::get(CGF.SizeTy, 0); 3095 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end(); 3096 ai != ae; ++ai, ++j) { 3097 llvm::Type *ArgTy = ai->getType(); 3098 if (Ops[j]->getType()->getPrimitiveSizeInBits() == 3099 ArgTy->getPrimitiveSizeInBits()) 3100 continue; 3101 3102 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy()); 3103 // The constant argument to an _n_ intrinsic always has Int32Ty, so truncate 3104 // it before inserting. 3105 Ops[j] = 3106 CGF.Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType()); 3107 Ops[j] = 3108 CGF.Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0); 3109 } 3110 3111 Value *Result = CGF.EmitNeonCall(F, Ops, s); 3112 llvm::Type *ResultType = CGF.ConvertType(E->getType()); 3113 if (ResultType->getPrimitiveSizeInBits() < 3114 Result->getType()->getPrimitiveSizeInBits()) 3115 return CGF.Builder.CreateExtractElement(Result, C0); 3116 3117 return CGF.Builder.CreateBitCast(Result, ResultType, s); 3118 } 3119 3120 Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( 3121 unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, 3122 const char *NameHint, unsigned Modifier, const CallExpr *E, 3123 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1) { 3124 // Get the last argument, which specifies the vector type. 3125 llvm::APSInt NeonTypeConst; 3126 const Expr *Arg = E->getArg(E->getNumArgs() - 1); 3127 if (!Arg->isIntegerConstantExpr(NeonTypeConst, getContext())) 3128 return nullptr; 3129 3130 // Determine the type of this overloaded NEON intrinsic. 3131 NeonTypeFlags Type(NeonTypeConst.getZExtValue()); 3132 bool Usgn = Type.isUnsigned(); 3133 bool Quad = Type.isQuad(); 3134 3135 llvm::VectorType *VTy = GetNeonType(this, Type); 3136 llvm::Type *Ty = VTy; 3137 if (!Ty) 3138 return nullptr; 3139 3140 auto getAlignmentValue32 = [&](Address addr) -> Value* { 3141 return Builder.getInt32(addr.getAlignment().getQuantity()); 3142 }; 3143 3144 unsigned Int = LLVMIntrinsic; 3145 if ((Modifier & UnsignedAlts) && !Usgn) 3146 Int = AltLLVMIntrinsic; 3147 3148 switch (BuiltinID) { 3149 default: break; 3150 case NEON::BI__builtin_neon_vabs_v: 3151 case NEON::BI__builtin_neon_vabsq_v: 3152 if (VTy->getElementType()->isFloatingPointTy()) 3153 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs"); 3154 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs"); 3155 case NEON::BI__builtin_neon_vaddhn_v: { 3156 llvm::VectorType *SrcTy = 3157 llvm::VectorType::getExtendedElementVectorType(VTy); 3158 3159 // %sum = add <4 x i32> %lhs, %rhs 3160 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 3161 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); 3162 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn"); 3163 3164 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16> 3165 Constant *ShiftAmt = 3166 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2); 3167 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn"); 3168 3169 // %res = trunc <4 x i32> %high to <4 x i16> 3170 return Builder.CreateTrunc(Ops[0], VTy, "vaddhn"); 3171 } 3172 case NEON::BI__builtin_neon_vcale_v: 3173 case NEON::BI__builtin_neon_vcaleq_v: 3174 case NEON::BI__builtin_neon_vcalt_v: 3175 case NEON::BI__builtin_neon_vcaltq_v: 3176 std::swap(Ops[0], Ops[1]); 3177 case NEON::BI__builtin_neon_vcage_v: 3178 case NEON::BI__builtin_neon_vcageq_v: 3179 case NEON::BI__builtin_neon_vcagt_v: 3180 case NEON::BI__builtin_neon_vcagtq_v: { 3181 llvm::Type *VecFlt = llvm::VectorType::get( 3182 VTy->getScalarSizeInBits() == 32 ? FloatTy : DoubleTy, 3183 VTy->getNumElements()); 3184 llvm::Type *Tys[] = { VTy, VecFlt }; 3185 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 3186 return EmitNeonCall(F, Ops, NameHint); 3187 } 3188 case NEON::BI__builtin_neon_vclz_v: 3189 case NEON::BI__builtin_neon_vclzq_v: 3190 // We generate target-independent intrinsic, which needs a second argument 3191 // for whether or not clz of zero is undefined; on ARM it isn't. 3192 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef())); 3193 break; 3194 case NEON::BI__builtin_neon_vcvt_f32_v: 3195 case NEON::BI__builtin_neon_vcvtq_f32_v: 3196 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3197 Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, Quad)); 3198 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") 3199 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); 3200 case NEON::BI__builtin_neon_vcvt_n_f32_v: 3201 case NEON::BI__builtin_neon_vcvt_n_f64_v: 3202 case NEON::BI__builtin_neon_vcvtq_n_f32_v: 3203 case NEON::BI__builtin_neon_vcvtq_n_f64_v: { 3204 llvm::Type *Tys[2] = { GetFloatNeonType(this, Type), Ty }; 3205 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic; 3206 Function *F = CGM.getIntrinsic(Int, Tys); 3207 return EmitNeonCall(F, Ops, "vcvt_n"); 3208 } 3209 case NEON::BI__builtin_neon_vcvt_n_s32_v: 3210 case NEON::BI__builtin_neon_vcvt_n_u32_v: 3211 case NEON::BI__builtin_neon_vcvt_n_s64_v: 3212 case NEON::BI__builtin_neon_vcvt_n_u64_v: 3213 case NEON::BI__builtin_neon_vcvtq_n_s32_v: 3214 case NEON::BI__builtin_neon_vcvtq_n_u32_v: 3215 case NEON::BI__builtin_neon_vcvtq_n_s64_v: 3216 case NEON::BI__builtin_neon_vcvtq_n_u64_v: { 3217 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 3218 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 3219 return EmitNeonCall(F, Ops, "vcvt_n"); 3220 } 3221 case NEON::BI__builtin_neon_vcvt_s32_v: 3222 case NEON::BI__builtin_neon_vcvt_u32_v: 3223 case NEON::BI__builtin_neon_vcvt_s64_v: 3224 case NEON::BI__builtin_neon_vcvt_u64_v: 3225 case NEON::BI__builtin_neon_vcvtq_s32_v: 3226 case NEON::BI__builtin_neon_vcvtq_u32_v: 3227 case NEON::BI__builtin_neon_vcvtq_s64_v: 3228 case NEON::BI__builtin_neon_vcvtq_u64_v: { 3229 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); 3230 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt") 3231 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt"); 3232 } 3233 case NEON::BI__builtin_neon_vcvta_s32_v: 3234 case NEON::BI__builtin_neon_vcvta_s64_v: 3235 case NEON::BI__builtin_neon_vcvta_u32_v: 3236 case NEON::BI__builtin_neon_vcvta_u64_v: 3237 case NEON::BI__builtin_neon_vcvtaq_s32_v: 3238 case NEON::BI__builtin_neon_vcvtaq_s64_v: 3239 case NEON::BI__builtin_neon_vcvtaq_u32_v: 3240 case NEON::BI__builtin_neon_vcvtaq_u64_v: 3241 case NEON::BI__builtin_neon_vcvtn_s32_v: 3242 case NEON::BI__builtin_neon_vcvtn_s64_v: 3243 case NEON::BI__builtin_neon_vcvtn_u32_v: 3244 case NEON::BI__builtin_neon_vcvtn_u64_v: 3245 case NEON::BI__builtin_neon_vcvtnq_s32_v: 3246 case NEON::BI__builtin_neon_vcvtnq_s64_v: 3247 case NEON::BI__builtin_neon_vcvtnq_u32_v: 3248 case NEON::BI__builtin_neon_vcvtnq_u64_v: 3249 case NEON::BI__builtin_neon_vcvtp_s32_v: 3250 case NEON::BI__builtin_neon_vcvtp_s64_v: 3251 case NEON::BI__builtin_neon_vcvtp_u32_v: 3252 case NEON::BI__builtin_neon_vcvtp_u64_v: 3253 case NEON::BI__builtin_neon_vcvtpq_s32_v: 3254 case NEON::BI__builtin_neon_vcvtpq_s64_v: 3255 case NEON::BI__builtin_neon_vcvtpq_u32_v: 3256 case NEON::BI__builtin_neon_vcvtpq_u64_v: 3257 case NEON::BI__builtin_neon_vcvtm_s32_v: 3258 case NEON::BI__builtin_neon_vcvtm_s64_v: 3259 case NEON::BI__builtin_neon_vcvtm_u32_v: 3260 case NEON::BI__builtin_neon_vcvtm_u64_v: 3261 case NEON::BI__builtin_neon_vcvtmq_s32_v: 3262 case NEON::BI__builtin_neon_vcvtmq_s64_v: 3263 case NEON::BI__builtin_neon_vcvtmq_u32_v: 3264 case NEON::BI__builtin_neon_vcvtmq_u64_v: { 3265 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 3266 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint); 3267 } 3268 case NEON::BI__builtin_neon_vext_v: 3269 case NEON::BI__builtin_neon_vextq_v: { 3270 int CV = cast<ConstantInt>(Ops[2])->getSExtValue(); 3271 SmallVector<int, 16> Indices; 3272 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 3273 Indices.push_back(i+CV); 3274 3275 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3276 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3277 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices, "vext"); 3278 } 3279 case NEON::BI__builtin_neon_vfma_v: 3280 case NEON::BI__builtin_neon_vfmaq_v: { 3281 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 3282 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3283 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3284 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 3285 3286 // NEON intrinsic puts accumulator first, unlike the LLVM fma. 3287 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); 3288 } 3289 case NEON::BI__builtin_neon_vld1_v: 3290 case NEON::BI__builtin_neon_vld1q_v: { 3291 llvm::Type *Tys[] = {Ty, Int8PtrTy}; 3292 Ops.push_back(getAlignmentValue32(PtrOp0)); 3293 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1"); 3294 } 3295 case NEON::BI__builtin_neon_vld2_v: 3296 case NEON::BI__builtin_neon_vld2q_v: 3297 case NEON::BI__builtin_neon_vld3_v: 3298 case NEON::BI__builtin_neon_vld3q_v: 3299 case NEON::BI__builtin_neon_vld4_v: 3300 case NEON::BI__builtin_neon_vld4q_v: { 3301 llvm::Type *Tys[] = {Ty, Int8PtrTy}; 3302 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 3303 Value *Align = getAlignmentValue32(PtrOp1); 3304 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint); 3305 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 3306 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3307 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 3308 } 3309 case NEON::BI__builtin_neon_vld1_dup_v: 3310 case NEON::BI__builtin_neon_vld1q_dup_v: { 3311 Value *V = UndefValue::get(Ty); 3312 Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 3313 PtrOp0 = Builder.CreateBitCast(PtrOp0, Ty); 3314 LoadInst *Ld = Builder.CreateLoad(PtrOp0); 3315 llvm::Constant *CI = ConstantInt::get(SizeTy, 0); 3316 Ops[0] = Builder.CreateInsertElement(V, Ld, CI); 3317 return EmitNeonSplat(Ops[0], CI); 3318 } 3319 case NEON::BI__builtin_neon_vld2_lane_v: 3320 case NEON::BI__builtin_neon_vld2q_lane_v: 3321 case NEON::BI__builtin_neon_vld3_lane_v: 3322 case NEON::BI__builtin_neon_vld3q_lane_v: 3323 case NEON::BI__builtin_neon_vld4_lane_v: 3324 case NEON::BI__builtin_neon_vld4q_lane_v: { 3325 llvm::Type *Tys[] = {Ty, Int8PtrTy}; 3326 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 3327 for (unsigned I = 2; I < Ops.size() - 1; ++I) 3328 Ops[I] = Builder.CreateBitCast(Ops[I], Ty); 3329 Ops.push_back(getAlignmentValue32(PtrOp1)); 3330 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint); 3331 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 3332 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3333 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 3334 } 3335 case NEON::BI__builtin_neon_vmovl_v: { 3336 llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy); 3337 Ops[0] = Builder.CreateBitCast(Ops[0], DTy); 3338 if (Usgn) 3339 return Builder.CreateZExt(Ops[0], Ty, "vmovl"); 3340 return Builder.CreateSExt(Ops[0], Ty, "vmovl"); 3341 } 3342 case NEON::BI__builtin_neon_vmovn_v: { 3343 llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy); 3344 Ops[0] = Builder.CreateBitCast(Ops[0], QTy); 3345 return Builder.CreateTrunc(Ops[0], Ty, "vmovn"); 3346 } 3347 case NEON::BI__builtin_neon_vmull_v: 3348 // FIXME: the integer vmull operations could be emitted in terms of pure 3349 // LLVM IR (2 exts followed by a mul). Unfortunately LLVM has a habit of 3350 // hoisting the exts outside loops. Until global ISel comes along that can 3351 // see through such movement this leads to bad CodeGen. So we need an 3352 // intrinsic for now. 3353 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls; 3354 Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int; 3355 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); 3356 case NEON::BI__builtin_neon_vpadal_v: 3357 case NEON::BI__builtin_neon_vpadalq_v: { 3358 // The source operand type has twice as many elements of half the size. 3359 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); 3360 llvm::Type *EltTy = 3361 llvm::IntegerType::get(getLLVMContext(), EltBits / 2); 3362 llvm::Type *NarrowTy = 3363 llvm::VectorType::get(EltTy, VTy->getNumElements() * 2); 3364 llvm::Type *Tys[2] = { Ty, NarrowTy }; 3365 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); 3366 } 3367 case NEON::BI__builtin_neon_vpaddl_v: 3368 case NEON::BI__builtin_neon_vpaddlq_v: { 3369 // The source operand type has twice as many elements of half the size. 3370 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); 3371 llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2); 3372 llvm::Type *NarrowTy = 3373 llvm::VectorType::get(EltTy, VTy->getNumElements() * 2); 3374 llvm::Type *Tys[2] = { Ty, NarrowTy }; 3375 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl"); 3376 } 3377 case NEON::BI__builtin_neon_vqdmlal_v: 3378 case NEON::BI__builtin_neon_vqdmlsl_v: { 3379 SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end()); 3380 Ops[1] = 3381 EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), MulOps, "vqdmlal"); 3382 Ops.resize(2); 3383 return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint); 3384 } 3385 case NEON::BI__builtin_neon_vqshl_n_v: 3386 case NEON::BI__builtin_neon_vqshlq_n_v: 3387 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n", 3388 1, false); 3389 case NEON::BI__builtin_neon_vqshlu_n_v: 3390 case NEON::BI__builtin_neon_vqshluq_n_v: 3391 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n", 3392 1, false); 3393 case NEON::BI__builtin_neon_vrecpe_v: 3394 case NEON::BI__builtin_neon_vrecpeq_v: 3395 case NEON::BI__builtin_neon_vrsqrte_v: 3396 case NEON::BI__builtin_neon_vrsqrteq_v: 3397 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic; 3398 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint); 3399 3400 case NEON::BI__builtin_neon_vrshr_n_v: 3401 case NEON::BI__builtin_neon_vrshrq_n_v: 3402 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", 3403 1, true); 3404 case NEON::BI__builtin_neon_vshl_n_v: 3405 case NEON::BI__builtin_neon_vshlq_n_v: 3406 Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false); 3407 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1], 3408 "vshl_n"); 3409 case NEON::BI__builtin_neon_vshll_n_v: { 3410 llvm::Type *SrcTy = llvm::VectorType::getTruncatedElementVectorType(VTy); 3411 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 3412 if (Usgn) 3413 Ops[0] = Builder.CreateZExt(Ops[0], VTy); 3414 else 3415 Ops[0] = Builder.CreateSExt(Ops[0], VTy); 3416 Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false); 3417 return Builder.CreateShl(Ops[0], Ops[1], "vshll_n"); 3418 } 3419 case NEON::BI__builtin_neon_vshrn_n_v: { 3420 llvm::Type *SrcTy = llvm::VectorType::getExtendedElementVectorType(VTy); 3421 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 3422 Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false); 3423 if (Usgn) 3424 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]); 3425 else 3426 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]); 3427 return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n"); 3428 } 3429 case NEON::BI__builtin_neon_vshr_n_v: 3430 case NEON::BI__builtin_neon_vshrq_n_v: 3431 return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n"); 3432 case NEON::BI__builtin_neon_vst1_v: 3433 case NEON::BI__builtin_neon_vst1q_v: 3434 case NEON::BI__builtin_neon_vst2_v: 3435 case NEON::BI__builtin_neon_vst2q_v: 3436 case NEON::BI__builtin_neon_vst3_v: 3437 case NEON::BI__builtin_neon_vst3q_v: 3438 case NEON::BI__builtin_neon_vst4_v: 3439 case NEON::BI__builtin_neon_vst4q_v: 3440 case NEON::BI__builtin_neon_vst2_lane_v: 3441 case NEON::BI__builtin_neon_vst2q_lane_v: 3442 case NEON::BI__builtin_neon_vst3_lane_v: 3443 case NEON::BI__builtin_neon_vst3q_lane_v: 3444 case NEON::BI__builtin_neon_vst4_lane_v: 3445 case NEON::BI__builtin_neon_vst4q_lane_v: { 3446 llvm::Type *Tys[] = {Int8PtrTy, Ty}; 3447 Ops.push_back(getAlignmentValue32(PtrOp0)); 3448 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, ""); 3449 } 3450 case NEON::BI__builtin_neon_vsubhn_v: { 3451 llvm::VectorType *SrcTy = 3452 llvm::VectorType::getExtendedElementVectorType(VTy); 3453 3454 // %sum = add <4 x i32> %lhs, %rhs 3455 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 3456 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); 3457 Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn"); 3458 3459 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16> 3460 Constant *ShiftAmt = 3461 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2); 3462 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn"); 3463 3464 // %res = trunc <4 x i32> %high to <4 x i16> 3465 return Builder.CreateTrunc(Ops[0], VTy, "vsubhn"); 3466 } 3467 case NEON::BI__builtin_neon_vtrn_v: 3468 case NEON::BI__builtin_neon_vtrnq_v: { 3469 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 3470 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3471 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 3472 Value *SV = nullptr; 3473 3474 for (unsigned vi = 0; vi != 2; ++vi) { 3475 SmallVector<int, 16> Indices; 3476 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 3477 Indices.push_back(i+vi); 3478 Indices.push_back(i+e+vi); 3479 } 3480 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 3481 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn"); 3482 SV = Builder.CreateDefaultAlignedStore(SV, Addr); 3483 } 3484 return SV; 3485 } 3486 case NEON::BI__builtin_neon_vtst_v: 3487 case NEON::BI__builtin_neon_vtstq_v: { 3488 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3489 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3490 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); 3491 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], 3492 ConstantAggregateZero::get(Ty)); 3493 return Builder.CreateSExt(Ops[0], Ty, "vtst"); 3494 } 3495 case NEON::BI__builtin_neon_vuzp_v: 3496 case NEON::BI__builtin_neon_vuzpq_v: { 3497 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 3498 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3499 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 3500 Value *SV = nullptr; 3501 3502 for (unsigned vi = 0; vi != 2; ++vi) { 3503 SmallVector<int, 16> Indices; 3504 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 3505 Indices.push_back(2*i+vi); 3506 3507 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 3508 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp"); 3509 SV = Builder.CreateDefaultAlignedStore(SV, Addr); 3510 } 3511 return SV; 3512 } 3513 case NEON::BI__builtin_neon_vzip_v: 3514 case NEON::BI__builtin_neon_vzipq_v: { 3515 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 3516 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3517 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 3518 Value *SV = nullptr; 3519 3520 for (unsigned vi = 0; vi != 2; ++vi) { 3521 SmallVector<int, 16> Indices; 3522 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 3523 Indices.push_back((i + vi*e) >> 1); 3524 Indices.push_back(((i + vi*e) >> 1)+e); 3525 } 3526 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 3527 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip"); 3528 SV = Builder.CreateDefaultAlignedStore(SV, Addr); 3529 } 3530 return SV; 3531 } 3532 } 3533 3534 assert(Int && "Expected valid intrinsic number"); 3535 3536 // Determine the type(s) of this overloaded AArch64 intrinsic. 3537 Function *F = LookupNeonLLVMIntrinsic(Int, Modifier, Ty, E); 3538 3539 Value *Result = EmitNeonCall(F, Ops, NameHint); 3540 llvm::Type *ResultType = ConvertType(E->getType()); 3541 // AArch64 intrinsic one-element vector type cast to 3542 // scalar type expected by the builtin 3543 return Builder.CreateBitCast(Result, ResultType, NameHint); 3544 } 3545 3546 Value *CodeGenFunction::EmitAArch64CompareBuiltinExpr( 3547 Value *Op, llvm::Type *Ty, const CmpInst::Predicate Fp, 3548 const CmpInst::Predicate Ip, const Twine &Name) { 3549 llvm::Type *OTy = Op->getType(); 3550 3551 // FIXME: this is utterly horrific. We should not be looking at previous 3552 // codegen context to find out what needs doing. Unfortunately TableGen 3553 // currently gives us exactly the same calls for vceqz_f32 and vceqz_s32 3554 // (etc). 3555 if (BitCastInst *BI = dyn_cast<BitCastInst>(Op)) 3556 OTy = BI->getOperand(0)->getType(); 3557 3558 Op = Builder.CreateBitCast(Op, OTy); 3559 if (OTy->getScalarType()->isFloatingPointTy()) { 3560 Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy)); 3561 } else { 3562 Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy)); 3563 } 3564 return Builder.CreateSExt(Op, Ty, Name); 3565 } 3566 3567 static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops, 3568 Value *ExtOp, Value *IndexOp, 3569 llvm::Type *ResTy, unsigned IntID, 3570 const char *Name) { 3571 SmallVector<Value *, 2> TblOps; 3572 if (ExtOp) 3573 TblOps.push_back(ExtOp); 3574 3575 // Build a vector containing sequential number like (0, 1, 2, ..., 15) 3576 SmallVector<int, 16> Indices; 3577 llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType()); 3578 for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) { 3579 Indices.push_back(2*i); 3580 Indices.push_back(2*i+1); 3581 } 3582 3583 int PairPos = 0, End = Ops.size() - 1; 3584 while (PairPos < End) { 3585 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], 3586 Ops[PairPos+1], Indices, 3587 Name)); 3588 PairPos += 2; 3589 } 3590 3591 // If there's an odd number of 64-bit lookup table, fill the high 64-bit 3592 // of the 128-bit lookup table with zero. 3593 if (PairPos == End) { 3594 Value *ZeroTbl = ConstantAggregateZero::get(TblTy); 3595 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], 3596 ZeroTbl, Indices, Name)); 3597 } 3598 3599 Function *TblF; 3600 TblOps.push_back(IndexOp); 3601 TblF = CGF.CGM.getIntrinsic(IntID, ResTy); 3602 3603 return CGF.EmitNeonCall(TblF, TblOps, Name); 3604 } 3605 3606 Value *CodeGenFunction::GetValueForARMHint(unsigned BuiltinID) { 3607 unsigned Value; 3608 switch (BuiltinID) { 3609 default: 3610 return nullptr; 3611 case ARM::BI__builtin_arm_nop: 3612 Value = 0; 3613 break; 3614 case ARM::BI__builtin_arm_yield: 3615 case ARM::BI__yield: 3616 Value = 1; 3617 break; 3618 case ARM::BI__builtin_arm_wfe: 3619 case ARM::BI__wfe: 3620 Value = 2; 3621 break; 3622 case ARM::BI__builtin_arm_wfi: 3623 case ARM::BI__wfi: 3624 Value = 3; 3625 break; 3626 case ARM::BI__builtin_arm_sev: 3627 case ARM::BI__sev: 3628 Value = 4; 3629 break; 3630 case ARM::BI__builtin_arm_sevl: 3631 case ARM::BI__sevl: 3632 Value = 5; 3633 break; 3634 } 3635 3636 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint), 3637 llvm::ConstantInt::get(Int32Ty, Value)); 3638 } 3639 3640 // Generates the IR for the read/write special register builtin, 3641 // ValueType is the type of the value that is to be written or read, 3642 // RegisterType is the type of the register being written to or read from. 3643 static Value *EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, 3644 const CallExpr *E, 3645 llvm::Type *RegisterType, 3646 llvm::Type *ValueType, bool IsRead) { 3647 // write and register intrinsics only support 32 and 64 bit operations. 3648 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64)) 3649 && "Unsupported size for register."); 3650 3651 CodeGen::CGBuilderTy &Builder = CGF.Builder; 3652 CodeGen::CodeGenModule &CGM = CGF.CGM; 3653 LLVMContext &Context = CGM.getLLVMContext(); 3654 3655 const Expr *SysRegStrExpr = E->getArg(0)->IgnoreParenCasts(); 3656 StringRef SysReg = cast<StringLiteral>(SysRegStrExpr)->getString(); 3657 3658 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) }; 3659 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); 3660 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName); 3661 3662 llvm::Type *Types[] = { RegisterType }; 3663 3664 bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32); 3665 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64)) 3666 && "Can't fit 64-bit value in 32-bit register"); 3667 3668 if (IsRead) { 3669 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types); 3670 llvm::Value *Call = Builder.CreateCall(F, Metadata); 3671 3672 if (MixedTypes) 3673 // Read into 64 bit register and then truncate result to 32 bit. 3674 return Builder.CreateTrunc(Call, ValueType); 3675 3676 if (ValueType->isPointerTy()) 3677 // Have i32/i64 result (Call) but want to return a VoidPtrTy (i8*). 3678 return Builder.CreateIntToPtr(Call, ValueType); 3679 3680 return Call; 3681 } 3682 3683 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types); 3684 llvm::Value *ArgValue = CGF.EmitScalarExpr(E->getArg(1)); 3685 if (MixedTypes) { 3686 // Extend 32 bit write value to 64 bit to pass to write. 3687 ArgValue = Builder.CreateZExt(ArgValue, RegisterType); 3688 return Builder.CreateCall(F, { Metadata, ArgValue }); 3689 } 3690 3691 if (ValueType->isPointerTy()) { 3692 // Have VoidPtrTy ArgValue but want to return an i32/i64. 3693 ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType); 3694 return Builder.CreateCall(F, { Metadata, ArgValue }); 3695 } 3696 3697 return Builder.CreateCall(F, { Metadata, ArgValue }); 3698 } 3699 3700 /// Return true if BuiltinID is an overloaded Neon intrinsic with an extra 3701 /// argument that specifies the vector type. 3702 static bool HasExtraNeonArgument(unsigned BuiltinID) { 3703 switch (BuiltinID) { 3704 default: break; 3705 case NEON::BI__builtin_neon_vget_lane_i8: 3706 case NEON::BI__builtin_neon_vget_lane_i16: 3707 case NEON::BI__builtin_neon_vget_lane_i32: 3708 case NEON::BI__builtin_neon_vget_lane_i64: 3709 case NEON::BI__builtin_neon_vget_lane_f32: 3710 case NEON::BI__builtin_neon_vgetq_lane_i8: 3711 case NEON::BI__builtin_neon_vgetq_lane_i16: 3712 case NEON::BI__builtin_neon_vgetq_lane_i32: 3713 case NEON::BI__builtin_neon_vgetq_lane_i64: 3714 case NEON::BI__builtin_neon_vgetq_lane_f32: 3715 case NEON::BI__builtin_neon_vset_lane_i8: 3716 case NEON::BI__builtin_neon_vset_lane_i16: 3717 case NEON::BI__builtin_neon_vset_lane_i32: 3718 case NEON::BI__builtin_neon_vset_lane_i64: 3719 case NEON::BI__builtin_neon_vset_lane_f32: 3720 case NEON::BI__builtin_neon_vsetq_lane_i8: 3721 case NEON::BI__builtin_neon_vsetq_lane_i16: 3722 case NEON::BI__builtin_neon_vsetq_lane_i32: 3723 case NEON::BI__builtin_neon_vsetq_lane_i64: 3724 case NEON::BI__builtin_neon_vsetq_lane_f32: 3725 case NEON::BI__builtin_neon_vsha1h_u32: 3726 case NEON::BI__builtin_neon_vsha1cq_u32: 3727 case NEON::BI__builtin_neon_vsha1pq_u32: 3728 case NEON::BI__builtin_neon_vsha1mq_u32: 3729 case ARM::BI_MoveToCoprocessor: 3730 case ARM::BI_MoveToCoprocessor2: 3731 return false; 3732 } 3733 return true; 3734 } 3735 3736 Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, 3737 const CallExpr *E) { 3738 if (auto Hint = GetValueForARMHint(BuiltinID)) 3739 return Hint; 3740 3741 if (BuiltinID == ARM::BI__emit) { 3742 bool IsThumb = getTarget().getTriple().getArch() == llvm::Triple::thumb; 3743 llvm::FunctionType *FTy = 3744 llvm::FunctionType::get(VoidTy, /*Variadic=*/false); 3745 3746 APSInt Value; 3747 if (!E->getArg(0)->EvaluateAsInt(Value, CGM.getContext())) 3748 llvm_unreachable("Sema will ensure that the parameter is constant"); 3749 3750 uint64_t ZExtValue = Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue(); 3751 3752 llvm::InlineAsm *Emit = 3753 IsThumb ? InlineAsm::get(FTy, ".inst.n 0x" + utohexstr(ZExtValue), "", 3754 /*SideEffects=*/true) 3755 : InlineAsm::get(FTy, ".inst 0x" + utohexstr(ZExtValue), "", 3756 /*SideEffects=*/true); 3757 3758 return Builder.CreateCall(Emit); 3759 } 3760 3761 if (BuiltinID == ARM::BI__builtin_arm_dbg) { 3762 Value *Option = EmitScalarExpr(E->getArg(0)); 3763 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option); 3764 } 3765 3766 if (BuiltinID == ARM::BI__builtin_arm_prefetch) { 3767 Value *Address = EmitScalarExpr(E->getArg(0)); 3768 Value *RW = EmitScalarExpr(E->getArg(1)); 3769 Value *IsData = EmitScalarExpr(E->getArg(2)); 3770 3771 // Locality is not supported on ARM target 3772 Value *Locality = llvm::ConstantInt::get(Int32Ty, 3); 3773 3774 Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 3775 return Builder.CreateCall(F, {Address, RW, Locality, IsData}); 3776 } 3777 3778 if (BuiltinID == ARM::BI__builtin_arm_rbit) { 3779 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_rbit), 3780 EmitScalarExpr(E->getArg(0)), 3781 "rbit"); 3782 } 3783 3784 if (BuiltinID == ARM::BI__clear_cache) { 3785 assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments"); 3786 const FunctionDecl *FD = E->getDirectCallee(); 3787 Value *Ops[2]; 3788 for (unsigned i = 0; i < 2; i++) 3789 Ops[i] = EmitScalarExpr(E->getArg(i)); 3790 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType()); 3791 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty); 3792 StringRef Name = FD->getName(); 3793 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); 3794 } 3795 3796 if (BuiltinID == ARM::BI__builtin_arm_ldrexd || 3797 ((BuiltinID == ARM::BI__builtin_arm_ldrex || 3798 BuiltinID == ARM::BI__builtin_arm_ldaex) && 3799 getContext().getTypeSize(E->getType()) == 64) || 3800 BuiltinID == ARM::BI__ldrexd) { 3801 Function *F; 3802 3803 switch (BuiltinID) { 3804 default: llvm_unreachable("unexpected builtin"); 3805 case ARM::BI__builtin_arm_ldaex: 3806 F = CGM.getIntrinsic(Intrinsic::arm_ldaexd); 3807 break; 3808 case ARM::BI__builtin_arm_ldrexd: 3809 case ARM::BI__builtin_arm_ldrex: 3810 case ARM::BI__ldrexd: 3811 F = CGM.getIntrinsic(Intrinsic::arm_ldrexd); 3812 break; 3813 } 3814 3815 Value *LdPtr = EmitScalarExpr(E->getArg(0)); 3816 Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy), 3817 "ldrexd"); 3818 3819 Value *Val0 = Builder.CreateExtractValue(Val, 1); 3820 Value *Val1 = Builder.CreateExtractValue(Val, 0); 3821 Val0 = Builder.CreateZExt(Val0, Int64Ty); 3822 Val1 = Builder.CreateZExt(Val1, Int64Ty); 3823 3824 Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32); 3825 Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */); 3826 Val = Builder.CreateOr(Val, Val1); 3827 return Builder.CreateBitCast(Val, ConvertType(E->getType())); 3828 } 3829 3830 if (BuiltinID == ARM::BI__builtin_arm_ldrex || 3831 BuiltinID == ARM::BI__builtin_arm_ldaex) { 3832 Value *LoadAddr = EmitScalarExpr(E->getArg(0)); 3833 3834 QualType Ty = E->getType(); 3835 llvm::Type *RealResTy = ConvertType(Ty); 3836 llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(), 3837 getContext().getTypeSize(Ty)); 3838 LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo()); 3839 3840 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_ldaex 3841 ? Intrinsic::arm_ldaex 3842 : Intrinsic::arm_ldrex, 3843 LoadAddr->getType()); 3844 Value *Val = Builder.CreateCall(F, LoadAddr, "ldrex"); 3845 3846 if (RealResTy->isPointerTy()) 3847 return Builder.CreateIntToPtr(Val, RealResTy); 3848 else { 3849 Val = Builder.CreateTruncOrBitCast(Val, IntResTy); 3850 return Builder.CreateBitCast(Val, RealResTy); 3851 } 3852 } 3853 3854 if (BuiltinID == ARM::BI__builtin_arm_strexd || 3855 ((BuiltinID == ARM::BI__builtin_arm_stlex || 3856 BuiltinID == ARM::BI__builtin_arm_strex) && 3857 getContext().getTypeSize(E->getArg(0)->getType()) == 64)) { 3858 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex 3859 ? Intrinsic::arm_stlexd 3860 : Intrinsic::arm_strexd); 3861 llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, nullptr); 3862 3863 Address Tmp = CreateMemTemp(E->getArg(0)->getType()); 3864 Value *Val = EmitScalarExpr(E->getArg(0)); 3865 Builder.CreateStore(Val, Tmp); 3866 3867 Address LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy)); 3868 Val = Builder.CreateLoad(LdPtr); 3869 3870 Value *Arg0 = Builder.CreateExtractValue(Val, 0); 3871 Value *Arg1 = Builder.CreateExtractValue(Val, 1); 3872 Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), Int8PtrTy); 3873 return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "strexd"); 3874 } 3875 3876 if (BuiltinID == ARM::BI__builtin_arm_strex || 3877 BuiltinID == ARM::BI__builtin_arm_stlex) { 3878 Value *StoreVal = EmitScalarExpr(E->getArg(0)); 3879 Value *StoreAddr = EmitScalarExpr(E->getArg(1)); 3880 3881 QualType Ty = E->getArg(0)->getType(); 3882 llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(), 3883 getContext().getTypeSize(Ty)); 3884 StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo()); 3885 3886 if (StoreVal->getType()->isPointerTy()) 3887 StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty); 3888 else { 3889 StoreVal = Builder.CreateBitCast(StoreVal, StoreTy); 3890 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty); 3891 } 3892 3893 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex 3894 ? Intrinsic::arm_stlex 3895 : Intrinsic::arm_strex, 3896 StoreAddr->getType()); 3897 return Builder.CreateCall(F, {StoreVal, StoreAddr}, "strex"); 3898 } 3899 3900 if (BuiltinID == ARM::BI__builtin_arm_clrex) { 3901 Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex); 3902 return Builder.CreateCall(F); 3903 } 3904 3905 // CRC32 3906 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic; 3907 switch (BuiltinID) { 3908 case ARM::BI__builtin_arm_crc32b: 3909 CRCIntrinsicID = Intrinsic::arm_crc32b; break; 3910 case ARM::BI__builtin_arm_crc32cb: 3911 CRCIntrinsicID = Intrinsic::arm_crc32cb; break; 3912 case ARM::BI__builtin_arm_crc32h: 3913 CRCIntrinsicID = Intrinsic::arm_crc32h; break; 3914 case ARM::BI__builtin_arm_crc32ch: 3915 CRCIntrinsicID = Intrinsic::arm_crc32ch; break; 3916 case ARM::BI__builtin_arm_crc32w: 3917 case ARM::BI__builtin_arm_crc32d: 3918 CRCIntrinsicID = Intrinsic::arm_crc32w; break; 3919 case ARM::BI__builtin_arm_crc32cw: 3920 case ARM::BI__builtin_arm_crc32cd: 3921 CRCIntrinsicID = Intrinsic::arm_crc32cw; break; 3922 } 3923 3924 if (CRCIntrinsicID != Intrinsic::not_intrinsic) { 3925 Value *Arg0 = EmitScalarExpr(E->getArg(0)); 3926 Value *Arg1 = EmitScalarExpr(E->getArg(1)); 3927 3928 // crc32{c,}d intrinsics are implemnted as two calls to crc32{c,}w 3929 // intrinsics, hence we need different codegen for these cases. 3930 if (BuiltinID == ARM::BI__builtin_arm_crc32d || 3931 BuiltinID == ARM::BI__builtin_arm_crc32cd) { 3932 Value *C1 = llvm::ConstantInt::get(Int64Ty, 32); 3933 Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty); 3934 Value *Arg1b = Builder.CreateLShr(Arg1, C1); 3935 Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty); 3936 3937 Function *F = CGM.getIntrinsic(CRCIntrinsicID); 3938 Value *Res = Builder.CreateCall(F, {Arg0, Arg1a}); 3939 return Builder.CreateCall(F, {Res, Arg1b}); 3940 } else { 3941 Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty); 3942 3943 Function *F = CGM.getIntrinsic(CRCIntrinsicID); 3944 return Builder.CreateCall(F, {Arg0, Arg1}); 3945 } 3946 } 3947 3948 if (BuiltinID == ARM::BI__builtin_arm_rsr || 3949 BuiltinID == ARM::BI__builtin_arm_rsr64 || 3950 BuiltinID == ARM::BI__builtin_arm_rsrp || 3951 BuiltinID == ARM::BI__builtin_arm_wsr || 3952 BuiltinID == ARM::BI__builtin_arm_wsr64 || 3953 BuiltinID == ARM::BI__builtin_arm_wsrp) { 3954 3955 bool IsRead = BuiltinID == ARM::BI__builtin_arm_rsr || 3956 BuiltinID == ARM::BI__builtin_arm_rsr64 || 3957 BuiltinID == ARM::BI__builtin_arm_rsrp; 3958 3959 bool IsPointerBuiltin = BuiltinID == ARM::BI__builtin_arm_rsrp || 3960 BuiltinID == ARM::BI__builtin_arm_wsrp; 3961 3962 bool Is64Bit = BuiltinID == ARM::BI__builtin_arm_rsr64 || 3963 BuiltinID == ARM::BI__builtin_arm_wsr64; 3964 3965 llvm::Type *ValueType; 3966 llvm::Type *RegisterType; 3967 if (IsPointerBuiltin) { 3968 ValueType = VoidPtrTy; 3969 RegisterType = Int32Ty; 3970 } else if (Is64Bit) { 3971 ValueType = RegisterType = Int64Ty; 3972 } else { 3973 ValueType = RegisterType = Int32Ty; 3974 } 3975 3976 return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType, IsRead); 3977 } 3978 3979 // Find out if any arguments are required to be integer constant 3980 // expressions. 3981 unsigned ICEArguments = 0; 3982 ASTContext::GetBuiltinTypeError Error; 3983 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 3984 assert(Error == ASTContext::GE_None && "Should not codegen an error"); 3985 3986 auto getAlignmentValue32 = [&](Address addr) -> Value* { 3987 return Builder.getInt32(addr.getAlignment().getQuantity()); 3988 }; 3989 3990 Address PtrOp0 = Address::invalid(); 3991 Address PtrOp1 = Address::invalid(); 3992 SmallVector<Value*, 4> Ops; 3993 bool HasExtraArg = HasExtraNeonArgument(BuiltinID); 3994 unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0); 3995 for (unsigned i = 0, e = NumArgs; i != e; i++) { 3996 if (i == 0) { 3997 switch (BuiltinID) { 3998 case NEON::BI__builtin_neon_vld1_v: 3999 case NEON::BI__builtin_neon_vld1q_v: 4000 case NEON::BI__builtin_neon_vld1q_lane_v: 4001 case NEON::BI__builtin_neon_vld1_lane_v: 4002 case NEON::BI__builtin_neon_vld1_dup_v: 4003 case NEON::BI__builtin_neon_vld1q_dup_v: 4004 case NEON::BI__builtin_neon_vst1_v: 4005 case NEON::BI__builtin_neon_vst1q_v: 4006 case NEON::BI__builtin_neon_vst1q_lane_v: 4007 case NEON::BI__builtin_neon_vst1_lane_v: 4008 case NEON::BI__builtin_neon_vst2_v: 4009 case NEON::BI__builtin_neon_vst2q_v: 4010 case NEON::BI__builtin_neon_vst2_lane_v: 4011 case NEON::BI__builtin_neon_vst2q_lane_v: 4012 case NEON::BI__builtin_neon_vst3_v: 4013 case NEON::BI__builtin_neon_vst3q_v: 4014 case NEON::BI__builtin_neon_vst3_lane_v: 4015 case NEON::BI__builtin_neon_vst3q_lane_v: 4016 case NEON::BI__builtin_neon_vst4_v: 4017 case NEON::BI__builtin_neon_vst4q_v: 4018 case NEON::BI__builtin_neon_vst4_lane_v: 4019 case NEON::BI__builtin_neon_vst4q_lane_v: 4020 // Get the alignment for the argument in addition to the value; 4021 // we'll use it later. 4022 PtrOp0 = EmitPointerWithAlignment(E->getArg(0)); 4023 Ops.push_back(PtrOp0.getPointer()); 4024 continue; 4025 } 4026 } 4027 if (i == 1) { 4028 switch (BuiltinID) { 4029 case NEON::BI__builtin_neon_vld2_v: 4030 case NEON::BI__builtin_neon_vld2q_v: 4031 case NEON::BI__builtin_neon_vld3_v: 4032 case NEON::BI__builtin_neon_vld3q_v: 4033 case NEON::BI__builtin_neon_vld4_v: 4034 case NEON::BI__builtin_neon_vld4q_v: 4035 case NEON::BI__builtin_neon_vld2_lane_v: 4036 case NEON::BI__builtin_neon_vld2q_lane_v: 4037 case NEON::BI__builtin_neon_vld3_lane_v: 4038 case NEON::BI__builtin_neon_vld3q_lane_v: 4039 case NEON::BI__builtin_neon_vld4_lane_v: 4040 case NEON::BI__builtin_neon_vld4q_lane_v: 4041 case NEON::BI__builtin_neon_vld2_dup_v: 4042 case NEON::BI__builtin_neon_vld3_dup_v: 4043 case NEON::BI__builtin_neon_vld4_dup_v: 4044 // Get the alignment for the argument in addition to the value; 4045 // we'll use it later. 4046 PtrOp1 = EmitPointerWithAlignment(E->getArg(1)); 4047 Ops.push_back(PtrOp1.getPointer()); 4048 continue; 4049 } 4050 } 4051 4052 if ((ICEArguments & (1 << i)) == 0) { 4053 Ops.push_back(EmitScalarExpr(E->getArg(i))); 4054 } else { 4055 // If this is required to be a constant, constant fold it so that we know 4056 // that the generated intrinsic gets a ConstantInt. 4057 llvm::APSInt Result; 4058 bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext()); 4059 assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst; 4060 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); 4061 } 4062 } 4063 4064 switch (BuiltinID) { 4065 default: break; 4066 4067 case NEON::BI__builtin_neon_vget_lane_i8: 4068 case NEON::BI__builtin_neon_vget_lane_i16: 4069 case NEON::BI__builtin_neon_vget_lane_i32: 4070 case NEON::BI__builtin_neon_vget_lane_i64: 4071 case NEON::BI__builtin_neon_vget_lane_f32: 4072 case NEON::BI__builtin_neon_vgetq_lane_i8: 4073 case NEON::BI__builtin_neon_vgetq_lane_i16: 4074 case NEON::BI__builtin_neon_vgetq_lane_i32: 4075 case NEON::BI__builtin_neon_vgetq_lane_i64: 4076 case NEON::BI__builtin_neon_vgetq_lane_f32: 4077 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane"); 4078 4079 case NEON::BI__builtin_neon_vset_lane_i8: 4080 case NEON::BI__builtin_neon_vset_lane_i16: 4081 case NEON::BI__builtin_neon_vset_lane_i32: 4082 case NEON::BI__builtin_neon_vset_lane_i64: 4083 case NEON::BI__builtin_neon_vset_lane_f32: 4084 case NEON::BI__builtin_neon_vsetq_lane_i8: 4085 case NEON::BI__builtin_neon_vsetq_lane_i16: 4086 case NEON::BI__builtin_neon_vsetq_lane_i32: 4087 case NEON::BI__builtin_neon_vsetq_lane_i64: 4088 case NEON::BI__builtin_neon_vsetq_lane_f32: 4089 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 4090 4091 case NEON::BI__builtin_neon_vsha1h_u32: 4092 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops, 4093 "vsha1h"); 4094 case NEON::BI__builtin_neon_vsha1cq_u32: 4095 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops, 4096 "vsha1h"); 4097 case NEON::BI__builtin_neon_vsha1pq_u32: 4098 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops, 4099 "vsha1h"); 4100 case NEON::BI__builtin_neon_vsha1mq_u32: 4101 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops, 4102 "vsha1h"); 4103 4104 // The ARM _MoveToCoprocessor builtins put the input register value as 4105 // the first argument, but the LLVM intrinsic expects it as the third one. 4106 case ARM::BI_MoveToCoprocessor: 4107 case ARM::BI_MoveToCoprocessor2: { 4108 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI_MoveToCoprocessor ? 4109 Intrinsic::arm_mcr : Intrinsic::arm_mcr2); 4110 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0], 4111 Ops[3], Ops[4], Ops[5]}); 4112 } 4113 } 4114 4115 // Get the last argument, which specifies the vector type. 4116 assert(HasExtraArg); 4117 llvm::APSInt Result; 4118 const Expr *Arg = E->getArg(E->getNumArgs()-1); 4119 if (!Arg->isIntegerConstantExpr(Result, getContext())) 4120 return nullptr; 4121 4122 if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f || 4123 BuiltinID == ARM::BI__builtin_arm_vcvtr_d) { 4124 // Determine the overloaded type of this builtin. 4125 llvm::Type *Ty; 4126 if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f) 4127 Ty = FloatTy; 4128 else 4129 Ty = DoubleTy; 4130 4131 // Determine whether this is an unsigned conversion or not. 4132 bool usgn = Result.getZExtValue() == 1; 4133 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr; 4134 4135 // Call the appropriate intrinsic. 4136 Function *F = CGM.getIntrinsic(Int, Ty); 4137 return Builder.CreateCall(F, Ops, "vcvtr"); 4138 } 4139 4140 // Determine the type of this overloaded NEON intrinsic. 4141 NeonTypeFlags Type(Result.getZExtValue()); 4142 bool usgn = Type.isUnsigned(); 4143 bool rightShift = false; 4144 4145 llvm::VectorType *VTy = GetNeonType(this, Type); 4146 llvm::Type *Ty = VTy; 4147 if (!Ty) 4148 return nullptr; 4149 4150 // Many NEON builtins have identical semantics and uses in ARM and 4151 // AArch64. Emit these in a single function. 4152 auto IntrinsicMap = makeArrayRef(ARMSIMDIntrinsicMap); 4153 const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap( 4154 IntrinsicMap, BuiltinID, NEONSIMDIntrinsicsProvenSorted); 4155 if (Builtin) 4156 return EmitCommonNeonBuiltinExpr( 4157 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic, 4158 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1); 4159 4160 unsigned Int; 4161 switch (BuiltinID) { 4162 default: return nullptr; 4163 case NEON::BI__builtin_neon_vld1q_lane_v: 4164 // Handle 64-bit integer elements as a special case. Use shuffles of 4165 // one-element vectors to avoid poor code for i64 in the backend. 4166 if (VTy->getElementType()->isIntegerTy(64)) { 4167 // Extract the other lane. 4168 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4169 uint32_t Lane = cast<ConstantInt>(Ops[2])->getZExtValue(); 4170 Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane)); 4171 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); 4172 // Load the value as a one-element vector. 4173 Ty = llvm::VectorType::get(VTy->getElementType(), 1); 4174 llvm::Type *Tys[] = {Ty, Int8PtrTy}; 4175 Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Tys); 4176 Value *Align = getAlignmentValue32(PtrOp0); 4177 Value *Ld = Builder.CreateCall(F, {Ops[0], Align}); 4178 // Combine them. 4179 uint32_t Indices[] = {1 - Lane, Lane}; 4180 SV = llvm::ConstantDataVector::get(getLLVMContext(), Indices); 4181 return Builder.CreateShuffleVector(Ops[1], Ld, SV, "vld1q_lane"); 4182 } 4183 // fall through 4184 case NEON::BI__builtin_neon_vld1_lane_v: { 4185 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4186 PtrOp0 = Builder.CreateElementBitCast(PtrOp0, VTy->getElementType()); 4187 Value *Ld = Builder.CreateLoad(PtrOp0); 4188 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane"); 4189 } 4190 case NEON::BI__builtin_neon_vld2_dup_v: 4191 case NEON::BI__builtin_neon_vld3_dup_v: 4192 case NEON::BI__builtin_neon_vld4_dup_v: { 4193 // Handle 64-bit elements as a special-case. There is no "dup" needed. 4194 if (VTy->getElementType()->getPrimitiveSizeInBits() == 64) { 4195 switch (BuiltinID) { 4196 case NEON::BI__builtin_neon_vld2_dup_v: 4197 Int = Intrinsic::arm_neon_vld2; 4198 break; 4199 case NEON::BI__builtin_neon_vld3_dup_v: 4200 Int = Intrinsic::arm_neon_vld3; 4201 break; 4202 case NEON::BI__builtin_neon_vld4_dup_v: 4203 Int = Intrinsic::arm_neon_vld4; 4204 break; 4205 default: llvm_unreachable("unknown vld_dup intrinsic?"); 4206 } 4207 llvm::Type *Tys[] = {Ty, Int8PtrTy}; 4208 Function *F = CGM.getIntrinsic(Int, Tys); 4209 llvm::Value *Align = getAlignmentValue32(PtrOp1); 4210 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, "vld_dup"); 4211 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 4212 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4213 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 4214 } 4215 switch (BuiltinID) { 4216 case NEON::BI__builtin_neon_vld2_dup_v: 4217 Int = Intrinsic::arm_neon_vld2lane; 4218 break; 4219 case NEON::BI__builtin_neon_vld3_dup_v: 4220 Int = Intrinsic::arm_neon_vld3lane; 4221 break; 4222 case NEON::BI__builtin_neon_vld4_dup_v: 4223 Int = Intrinsic::arm_neon_vld4lane; 4224 break; 4225 default: llvm_unreachable("unknown vld_dup intrinsic?"); 4226 } 4227 llvm::Type *Tys[] = {Ty, Int8PtrTy}; 4228 Function *F = CGM.getIntrinsic(Int, Tys); 4229 llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType()); 4230 4231 SmallVector<Value*, 6> Args; 4232 Args.push_back(Ops[1]); 4233 Args.append(STy->getNumElements(), UndefValue::get(Ty)); 4234 4235 llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); 4236 Args.push_back(CI); 4237 Args.push_back(getAlignmentValue32(PtrOp1)); 4238 4239 Ops[1] = Builder.CreateCall(F, Args, "vld_dup"); 4240 // splat lane 0 to all elts in each vector of the result. 4241 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { 4242 Value *Val = Builder.CreateExtractValue(Ops[1], i); 4243 Value *Elt = Builder.CreateBitCast(Val, Ty); 4244 Elt = EmitNeonSplat(Elt, CI); 4245 Elt = Builder.CreateBitCast(Elt, Val->getType()); 4246 Ops[1] = Builder.CreateInsertValue(Ops[1], Elt, i); 4247 } 4248 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 4249 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4250 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 4251 } 4252 case NEON::BI__builtin_neon_vqrshrn_n_v: 4253 Int = 4254 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns; 4255 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n", 4256 1, true); 4257 case NEON::BI__builtin_neon_vqrshrun_n_v: 4258 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty), 4259 Ops, "vqrshrun_n", 1, true); 4260 case NEON::BI__builtin_neon_vqshrn_n_v: 4261 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns; 4262 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n", 4263 1, true); 4264 case NEON::BI__builtin_neon_vqshrun_n_v: 4265 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty), 4266 Ops, "vqshrun_n", 1, true); 4267 case NEON::BI__builtin_neon_vrecpe_v: 4268 case NEON::BI__builtin_neon_vrecpeq_v: 4269 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty), 4270 Ops, "vrecpe"); 4271 case NEON::BI__builtin_neon_vrshrn_n_v: 4272 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty), 4273 Ops, "vrshrn_n", 1, true); 4274 case NEON::BI__builtin_neon_vrsra_n_v: 4275 case NEON::BI__builtin_neon_vrsraq_n_v: 4276 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4277 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4278 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true); 4279 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts; 4280 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]}); 4281 return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n"); 4282 case NEON::BI__builtin_neon_vsri_n_v: 4283 case NEON::BI__builtin_neon_vsriq_n_v: 4284 rightShift = true; 4285 case NEON::BI__builtin_neon_vsli_n_v: 4286 case NEON::BI__builtin_neon_vsliq_n_v: 4287 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift); 4288 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty), 4289 Ops, "vsli_n"); 4290 case NEON::BI__builtin_neon_vsra_n_v: 4291 case NEON::BI__builtin_neon_vsraq_n_v: 4292 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4293 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); 4294 return Builder.CreateAdd(Ops[0], Ops[1]); 4295 case NEON::BI__builtin_neon_vst1q_lane_v: 4296 // Handle 64-bit integer elements as a special case. Use a shuffle to get 4297 // a one-element vector and avoid poor code for i64 in the backend. 4298 if (VTy->getElementType()->isIntegerTy(64)) { 4299 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4300 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2])); 4301 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); 4302 Ops[2] = getAlignmentValue32(PtrOp0); 4303 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()}; 4304 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1, 4305 Tys), Ops); 4306 } 4307 // fall through 4308 case NEON::BI__builtin_neon_vst1_lane_v: { 4309 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4310 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); 4311 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 4312 auto St = Builder.CreateStore(Ops[1], Builder.CreateBitCast(PtrOp0, Ty)); 4313 return St; 4314 } 4315 case NEON::BI__builtin_neon_vtbl1_v: 4316 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1), 4317 Ops, "vtbl1"); 4318 case NEON::BI__builtin_neon_vtbl2_v: 4319 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2), 4320 Ops, "vtbl2"); 4321 case NEON::BI__builtin_neon_vtbl3_v: 4322 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3), 4323 Ops, "vtbl3"); 4324 case NEON::BI__builtin_neon_vtbl4_v: 4325 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4), 4326 Ops, "vtbl4"); 4327 case NEON::BI__builtin_neon_vtbx1_v: 4328 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1), 4329 Ops, "vtbx1"); 4330 case NEON::BI__builtin_neon_vtbx2_v: 4331 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2), 4332 Ops, "vtbx2"); 4333 case NEON::BI__builtin_neon_vtbx3_v: 4334 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3), 4335 Ops, "vtbx3"); 4336 case NEON::BI__builtin_neon_vtbx4_v: 4337 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4), 4338 Ops, "vtbx4"); 4339 } 4340 } 4341 4342 static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, 4343 const CallExpr *E, 4344 SmallVectorImpl<Value *> &Ops) { 4345 unsigned int Int = 0; 4346 const char *s = nullptr; 4347 4348 switch (BuiltinID) { 4349 default: 4350 return nullptr; 4351 case NEON::BI__builtin_neon_vtbl1_v: 4352 case NEON::BI__builtin_neon_vqtbl1_v: 4353 case NEON::BI__builtin_neon_vqtbl1q_v: 4354 case NEON::BI__builtin_neon_vtbl2_v: 4355 case NEON::BI__builtin_neon_vqtbl2_v: 4356 case NEON::BI__builtin_neon_vqtbl2q_v: 4357 case NEON::BI__builtin_neon_vtbl3_v: 4358 case NEON::BI__builtin_neon_vqtbl3_v: 4359 case NEON::BI__builtin_neon_vqtbl3q_v: 4360 case NEON::BI__builtin_neon_vtbl4_v: 4361 case NEON::BI__builtin_neon_vqtbl4_v: 4362 case NEON::BI__builtin_neon_vqtbl4q_v: 4363 break; 4364 case NEON::BI__builtin_neon_vtbx1_v: 4365 case NEON::BI__builtin_neon_vqtbx1_v: 4366 case NEON::BI__builtin_neon_vqtbx1q_v: 4367 case NEON::BI__builtin_neon_vtbx2_v: 4368 case NEON::BI__builtin_neon_vqtbx2_v: 4369 case NEON::BI__builtin_neon_vqtbx2q_v: 4370 case NEON::BI__builtin_neon_vtbx3_v: 4371 case NEON::BI__builtin_neon_vqtbx3_v: 4372 case NEON::BI__builtin_neon_vqtbx3q_v: 4373 case NEON::BI__builtin_neon_vtbx4_v: 4374 case NEON::BI__builtin_neon_vqtbx4_v: 4375 case NEON::BI__builtin_neon_vqtbx4q_v: 4376 break; 4377 } 4378 4379 assert(E->getNumArgs() >= 3); 4380 4381 // Get the last argument, which specifies the vector type. 4382 llvm::APSInt Result; 4383 const Expr *Arg = E->getArg(E->getNumArgs() - 1); 4384 if (!Arg->isIntegerConstantExpr(Result, CGF.getContext())) 4385 return nullptr; 4386 4387 // Determine the type of this overloaded NEON intrinsic. 4388 NeonTypeFlags Type(Result.getZExtValue()); 4389 llvm::VectorType *Ty = GetNeonType(&CGF, Type); 4390 if (!Ty) 4391 return nullptr; 4392 4393 CodeGen::CGBuilderTy &Builder = CGF.Builder; 4394 4395 // AArch64 scalar builtins are not overloaded, they do not have an extra 4396 // argument that specifies the vector type, need to handle each case. 4397 switch (BuiltinID) { 4398 case NEON::BI__builtin_neon_vtbl1_v: { 4399 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 1), nullptr, 4400 Ops[1], Ty, Intrinsic::aarch64_neon_tbl1, 4401 "vtbl1"); 4402 } 4403 case NEON::BI__builtin_neon_vtbl2_v: { 4404 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 2), nullptr, 4405 Ops[2], Ty, Intrinsic::aarch64_neon_tbl1, 4406 "vtbl1"); 4407 } 4408 case NEON::BI__builtin_neon_vtbl3_v: { 4409 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 3), nullptr, 4410 Ops[3], Ty, Intrinsic::aarch64_neon_tbl2, 4411 "vtbl2"); 4412 } 4413 case NEON::BI__builtin_neon_vtbl4_v: { 4414 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 4), nullptr, 4415 Ops[4], Ty, Intrinsic::aarch64_neon_tbl2, 4416 "vtbl2"); 4417 } 4418 case NEON::BI__builtin_neon_vtbx1_v: { 4419 Value *TblRes = 4420 packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 1), nullptr, Ops[2], 4421 Ty, Intrinsic::aarch64_neon_tbl1, "vtbl1"); 4422 4423 llvm::Constant *EightV = ConstantInt::get(Ty, 8); 4424 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV); 4425 CmpRes = Builder.CreateSExt(CmpRes, Ty); 4426 4427 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); 4428 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes); 4429 return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx"); 4430 } 4431 case NEON::BI__builtin_neon_vtbx2_v: { 4432 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 2), Ops[0], 4433 Ops[3], Ty, Intrinsic::aarch64_neon_tbx1, 4434 "vtbx1"); 4435 } 4436 case NEON::BI__builtin_neon_vtbx3_v: { 4437 Value *TblRes = 4438 packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 3), nullptr, Ops[4], 4439 Ty, Intrinsic::aarch64_neon_tbl2, "vtbl2"); 4440 4441 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24); 4442 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4], 4443 TwentyFourV); 4444 CmpRes = Builder.CreateSExt(CmpRes, Ty); 4445 4446 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); 4447 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes); 4448 return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx"); 4449 } 4450 case NEON::BI__builtin_neon_vtbx4_v: { 4451 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 4), Ops[0], 4452 Ops[5], Ty, Intrinsic::aarch64_neon_tbx2, 4453 "vtbx2"); 4454 } 4455 case NEON::BI__builtin_neon_vqtbl1_v: 4456 case NEON::BI__builtin_neon_vqtbl1q_v: 4457 Int = Intrinsic::aarch64_neon_tbl1; s = "vtbl1"; break; 4458 case NEON::BI__builtin_neon_vqtbl2_v: 4459 case NEON::BI__builtin_neon_vqtbl2q_v: { 4460 Int = Intrinsic::aarch64_neon_tbl2; s = "vtbl2"; break; 4461 case NEON::BI__builtin_neon_vqtbl3_v: 4462 case NEON::BI__builtin_neon_vqtbl3q_v: 4463 Int = Intrinsic::aarch64_neon_tbl3; s = "vtbl3"; break; 4464 case NEON::BI__builtin_neon_vqtbl4_v: 4465 case NEON::BI__builtin_neon_vqtbl4q_v: 4466 Int = Intrinsic::aarch64_neon_tbl4; s = "vtbl4"; break; 4467 case NEON::BI__builtin_neon_vqtbx1_v: 4468 case NEON::BI__builtin_neon_vqtbx1q_v: 4469 Int = Intrinsic::aarch64_neon_tbx1; s = "vtbx1"; break; 4470 case NEON::BI__builtin_neon_vqtbx2_v: 4471 case NEON::BI__builtin_neon_vqtbx2q_v: 4472 Int = Intrinsic::aarch64_neon_tbx2; s = "vtbx2"; break; 4473 case NEON::BI__builtin_neon_vqtbx3_v: 4474 case NEON::BI__builtin_neon_vqtbx3q_v: 4475 Int = Intrinsic::aarch64_neon_tbx3; s = "vtbx3"; break; 4476 case NEON::BI__builtin_neon_vqtbx4_v: 4477 case NEON::BI__builtin_neon_vqtbx4q_v: 4478 Int = Intrinsic::aarch64_neon_tbx4; s = "vtbx4"; break; 4479 } 4480 } 4481 4482 if (!Int) 4483 return nullptr; 4484 4485 Function *F = CGF.CGM.getIntrinsic(Int, Ty); 4486 return CGF.EmitNeonCall(F, Ops, s); 4487 } 4488 4489 Value *CodeGenFunction::vectorWrapScalar16(Value *Op) { 4490 llvm::Type *VTy = llvm::VectorType::get(Int16Ty, 4); 4491 Op = Builder.CreateBitCast(Op, Int16Ty); 4492 Value *V = UndefValue::get(VTy); 4493 llvm::Constant *CI = ConstantInt::get(SizeTy, 0); 4494 Op = Builder.CreateInsertElement(V, Op, CI); 4495 return Op; 4496 } 4497 4498 Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, 4499 const CallExpr *E) { 4500 unsigned HintID = static_cast<unsigned>(-1); 4501 switch (BuiltinID) { 4502 default: break; 4503 case AArch64::BI__builtin_arm_nop: 4504 HintID = 0; 4505 break; 4506 case AArch64::BI__builtin_arm_yield: 4507 HintID = 1; 4508 break; 4509 case AArch64::BI__builtin_arm_wfe: 4510 HintID = 2; 4511 break; 4512 case AArch64::BI__builtin_arm_wfi: 4513 HintID = 3; 4514 break; 4515 case AArch64::BI__builtin_arm_sev: 4516 HintID = 4; 4517 break; 4518 case AArch64::BI__builtin_arm_sevl: 4519 HintID = 5; 4520 break; 4521 } 4522 4523 if (HintID != static_cast<unsigned>(-1)) { 4524 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hint); 4525 return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID)); 4526 } 4527 4528 if (BuiltinID == AArch64::BI__builtin_arm_prefetch) { 4529 Value *Address = EmitScalarExpr(E->getArg(0)); 4530 Value *RW = EmitScalarExpr(E->getArg(1)); 4531 Value *CacheLevel = EmitScalarExpr(E->getArg(2)); 4532 Value *RetentionPolicy = EmitScalarExpr(E->getArg(3)); 4533 Value *IsData = EmitScalarExpr(E->getArg(4)); 4534 4535 Value *Locality = nullptr; 4536 if (cast<llvm::ConstantInt>(RetentionPolicy)->isZero()) { 4537 // Temporal fetch, needs to convert cache level to locality. 4538 Locality = llvm::ConstantInt::get(Int32Ty, 4539 -cast<llvm::ConstantInt>(CacheLevel)->getValue() + 3); 4540 } else { 4541 // Streaming fetch. 4542 Locality = llvm::ConstantInt::get(Int32Ty, 0); 4543 } 4544 4545 // FIXME: We need AArch64 specific LLVM intrinsic if we want to specify 4546 // PLDL3STRM or PLDL2STRM. 4547 Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 4548 return Builder.CreateCall(F, {Address, RW, Locality, IsData}); 4549 } 4550 4551 if (BuiltinID == AArch64::BI__builtin_arm_rbit) { 4552 assert((getContext().getTypeSize(E->getType()) == 32) && 4553 "rbit of unusual size!"); 4554 llvm::Value *Arg = EmitScalarExpr(E->getArg(0)); 4555 return Builder.CreateCall( 4556 CGM.getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg, "rbit"); 4557 } 4558 if (BuiltinID == AArch64::BI__builtin_arm_rbit64) { 4559 assert((getContext().getTypeSize(E->getType()) == 64) && 4560 "rbit of unusual size!"); 4561 llvm::Value *Arg = EmitScalarExpr(E->getArg(0)); 4562 return Builder.CreateCall( 4563 CGM.getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg, "rbit"); 4564 } 4565 4566 if (BuiltinID == AArch64::BI__clear_cache) { 4567 assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments"); 4568 const FunctionDecl *FD = E->getDirectCallee(); 4569 Value *Ops[2]; 4570 for (unsigned i = 0; i < 2; i++) 4571 Ops[i] = EmitScalarExpr(E->getArg(i)); 4572 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType()); 4573 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty); 4574 StringRef Name = FD->getName(); 4575 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); 4576 } 4577 4578 if ((BuiltinID == AArch64::BI__builtin_arm_ldrex || 4579 BuiltinID == AArch64::BI__builtin_arm_ldaex) && 4580 getContext().getTypeSize(E->getType()) == 128) { 4581 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex 4582 ? Intrinsic::aarch64_ldaxp 4583 : Intrinsic::aarch64_ldxp); 4584 4585 Value *LdPtr = EmitScalarExpr(E->getArg(0)); 4586 Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy), 4587 "ldxp"); 4588 4589 Value *Val0 = Builder.CreateExtractValue(Val, 1); 4590 Value *Val1 = Builder.CreateExtractValue(Val, 0); 4591 llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128); 4592 Val0 = Builder.CreateZExt(Val0, Int128Ty); 4593 Val1 = Builder.CreateZExt(Val1, Int128Ty); 4594 4595 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64); 4596 Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */); 4597 Val = Builder.CreateOr(Val, Val1); 4598 return Builder.CreateBitCast(Val, ConvertType(E->getType())); 4599 } else if (BuiltinID == AArch64::BI__builtin_arm_ldrex || 4600 BuiltinID == AArch64::BI__builtin_arm_ldaex) { 4601 Value *LoadAddr = EmitScalarExpr(E->getArg(0)); 4602 4603 QualType Ty = E->getType(); 4604 llvm::Type *RealResTy = ConvertType(Ty); 4605 llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(), 4606 getContext().getTypeSize(Ty)); 4607 LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo()); 4608 4609 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex 4610 ? Intrinsic::aarch64_ldaxr 4611 : Intrinsic::aarch64_ldxr, 4612 LoadAddr->getType()); 4613 Value *Val = Builder.CreateCall(F, LoadAddr, "ldxr"); 4614 4615 if (RealResTy->isPointerTy()) 4616 return Builder.CreateIntToPtr(Val, RealResTy); 4617 4618 Val = Builder.CreateTruncOrBitCast(Val, IntResTy); 4619 return Builder.CreateBitCast(Val, RealResTy); 4620 } 4621 4622 if ((BuiltinID == AArch64::BI__builtin_arm_strex || 4623 BuiltinID == AArch64::BI__builtin_arm_stlex) && 4624 getContext().getTypeSize(E->getArg(0)->getType()) == 128) { 4625 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex 4626 ? Intrinsic::aarch64_stlxp 4627 : Intrinsic::aarch64_stxp); 4628 llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty, nullptr); 4629 4630 Address Tmp = CreateMemTemp(E->getArg(0)->getType()); 4631 EmitAnyExprToMem(E->getArg(0), Tmp, Qualifiers(), /*init*/ true); 4632 4633 Tmp = Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(STy)); 4634 llvm::Value *Val = Builder.CreateLoad(Tmp); 4635 4636 Value *Arg0 = Builder.CreateExtractValue(Val, 0); 4637 Value *Arg1 = Builder.CreateExtractValue(Val, 1); 4638 Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), 4639 Int8PtrTy); 4640 return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "stxp"); 4641 } 4642 4643 if (BuiltinID == AArch64::BI__builtin_arm_strex || 4644 BuiltinID == AArch64::BI__builtin_arm_stlex) { 4645 Value *StoreVal = EmitScalarExpr(E->getArg(0)); 4646 Value *StoreAddr = EmitScalarExpr(E->getArg(1)); 4647 4648 QualType Ty = E->getArg(0)->getType(); 4649 llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(), 4650 getContext().getTypeSize(Ty)); 4651 StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo()); 4652 4653 if (StoreVal->getType()->isPointerTy()) 4654 StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty); 4655 else { 4656 StoreVal = Builder.CreateBitCast(StoreVal, StoreTy); 4657 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty); 4658 } 4659 4660 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex 4661 ? Intrinsic::aarch64_stlxr 4662 : Intrinsic::aarch64_stxr, 4663 StoreAddr->getType()); 4664 return Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr"); 4665 } 4666 4667 if (BuiltinID == AArch64::BI__builtin_arm_clrex) { 4668 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex); 4669 return Builder.CreateCall(F); 4670 } 4671 4672 // CRC32 4673 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic; 4674 switch (BuiltinID) { 4675 case AArch64::BI__builtin_arm_crc32b: 4676 CRCIntrinsicID = Intrinsic::aarch64_crc32b; break; 4677 case AArch64::BI__builtin_arm_crc32cb: 4678 CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break; 4679 case AArch64::BI__builtin_arm_crc32h: 4680 CRCIntrinsicID = Intrinsic::aarch64_crc32h; break; 4681 case AArch64::BI__builtin_arm_crc32ch: 4682 CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break; 4683 case AArch64::BI__builtin_arm_crc32w: 4684 CRCIntrinsicID = Intrinsic::aarch64_crc32w; break; 4685 case AArch64::BI__builtin_arm_crc32cw: 4686 CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break; 4687 case AArch64::BI__builtin_arm_crc32d: 4688 CRCIntrinsicID = Intrinsic::aarch64_crc32x; break; 4689 case AArch64::BI__builtin_arm_crc32cd: 4690 CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break; 4691 } 4692 4693 if (CRCIntrinsicID != Intrinsic::not_intrinsic) { 4694 Value *Arg0 = EmitScalarExpr(E->getArg(0)); 4695 Value *Arg1 = EmitScalarExpr(E->getArg(1)); 4696 Function *F = CGM.getIntrinsic(CRCIntrinsicID); 4697 4698 llvm::Type *DataTy = F->getFunctionType()->getParamType(1); 4699 Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy); 4700 4701 return Builder.CreateCall(F, {Arg0, Arg1}); 4702 } 4703 4704 if (BuiltinID == AArch64::BI__builtin_arm_rsr || 4705 BuiltinID == AArch64::BI__builtin_arm_rsr64 || 4706 BuiltinID == AArch64::BI__builtin_arm_rsrp || 4707 BuiltinID == AArch64::BI__builtin_arm_wsr || 4708 BuiltinID == AArch64::BI__builtin_arm_wsr64 || 4709 BuiltinID == AArch64::BI__builtin_arm_wsrp) { 4710 4711 bool IsRead = BuiltinID == AArch64::BI__builtin_arm_rsr || 4712 BuiltinID == AArch64::BI__builtin_arm_rsr64 || 4713 BuiltinID == AArch64::BI__builtin_arm_rsrp; 4714 4715 bool IsPointerBuiltin = BuiltinID == AArch64::BI__builtin_arm_rsrp || 4716 BuiltinID == AArch64::BI__builtin_arm_wsrp; 4717 4718 bool Is64Bit = BuiltinID != AArch64::BI__builtin_arm_rsr && 4719 BuiltinID != AArch64::BI__builtin_arm_wsr; 4720 4721 llvm::Type *ValueType; 4722 llvm::Type *RegisterType = Int64Ty; 4723 if (IsPointerBuiltin) { 4724 ValueType = VoidPtrTy; 4725 } else if (Is64Bit) { 4726 ValueType = Int64Ty; 4727 } else { 4728 ValueType = Int32Ty; 4729 } 4730 4731 return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType, IsRead); 4732 } 4733 4734 // Find out if any arguments are required to be integer constant 4735 // expressions. 4736 unsigned ICEArguments = 0; 4737 ASTContext::GetBuiltinTypeError Error; 4738 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 4739 assert(Error == ASTContext::GE_None && "Should not codegen an error"); 4740 4741 llvm::SmallVector<Value*, 4> Ops; 4742 for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) { 4743 if ((ICEArguments & (1 << i)) == 0) { 4744 Ops.push_back(EmitScalarExpr(E->getArg(i))); 4745 } else { 4746 // If this is required to be a constant, constant fold it so that we know 4747 // that the generated intrinsic gets a ConstantInt. 4748 llvm::APSInt Result; 4749 bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext()); 4750 assert(IsConst && "Constant arg isn't actually constant?"); 4751 (void)IsConst; 4752 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); 4753 } 4754 } 4755 4756 auto SISDMap = makeArrayRef(AArch64SISDIntrinsicMap); 4757 const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap( 4758 SISDMap, BuiltinID, AArch64SISDIntrinsicsProvenSorted); 4759 4760 if (Builtin) { 4761 Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1))); 4762 Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E); 4763 assert(Result && "SISD intrinsic should have been handled"); 4764 return Result; 4765 } 4766 4767 llvm::APSInt Result; 4768 const Expr *Arg = E->getArg(E->getNumArgs()-1); 4769 NeonTypeFlags Type(0); 4770 if (Arg->isIntegerConstantExpr(Result, getContext())) 4771 // Determine the type of this overloaded NEON intrinsic. 4772 Type = NeonTypeFlags(Result.getZExtValue()); 4773 4774 bool usgn = Type.isUnsigned(); 4775 bool quad = Type.isQuad(); 4776 4777 // Handle non-overloaded intrinsics first. 4778 switch (BuiltinID) { 4779 default: break; 4780 case NEON::BI__builtin_neon_vldrq_p128: { 4781 llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128); 4782 Value *Ptr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int128PTy); 4783 return Builder.CreateDefaultAlignedLoad(Ptr); 4784 } 4785 case NEON::BI__builtin_neon_vstrq_p128: { 4786 llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128); 4787 Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy); 4788 return Builder.CreateDefaultAlignedStore(EmitScalarExpr(E->getArg(1)), Ptr); 4789 } 4790 case NEON::BI__builtin_neon_vcvts_u32_f32: 4791 case NEON::BI__builtin_neon_vcvtd_u64_f64: 4792 usgn = true; 4793 // FALL THROUGH 4794 case NEON::BI__builtin_neon_vcvts_s32_f32: 4795 case NEON::BI__builtin_neon_vcvtd_s64_f64: { 4796 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4797 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; 4798 llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty; 4799 llvm::Type *FTy = Is64 ? DoubleTy : FloatTy; 4800 Ops[0] = Builder.CreateBitCast(Ops[0], FTy); 4801 if (usgn) 4802 return Builder.CreateFPToUI(Ops[0], InTy); 4803 return Builder.CreateFPToSI(Ops[0], InTy); 4804 } 4805 case NEON::BI__builtin_neon_vcvts_f32_u32: 4806 case NEON::BI__builtin_neon_vcvtd_f64_u64: 4807 usgn = true; 4808 // FALL THROUGH 4809 case NEON::BI__builtin_neon_vcvts_f32_s32: 4810 case NEON::BI__builtin_neon_vcvtd_f64_s64: { 4811 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4812 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; 4813 llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty; 4814 llvm::Type *FTy = Is64 ? DoubleTy : FloatTy; 4815 Ops[0] = Builder.CreateBitCast(Ops[0], InTy); 4816 if (usgn) 4817 return Builder.CreateUIToFP(Ops[0], FTy); 4818 return Builder.CreateSIToFP(Ops[0], FTy); 4819 } 4820 case NEON::BI__builtin_neon_vpaddd_s64: { 4821 llvm::Type *Ty = llvm::VectorType::get(Int64Ty, 2); 4822 Value *Vec = EmitScalarExpr(E->getArg(0)); 4823 // The vector is v2f64, so make sure it's bitcast to that. 4824 Vec = Builder.CreateBitCast(Vec, Ty, "v2i64"); 4825 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); 4826 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); 4827 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 4828 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 4829 // Pairwise addition of a v2f64 into a scalar f64. 4830 return Builder.CreateAdd(Op0, Op1, "vpaddd"); 4831 } 4832 case NEON::BI__builtin_neon_vpaddd_f64: { 4833 llvm::Type *Ty = 4834 llvm::VectorType::get(DoubleTy, 2); 4835 Value *Vec = EmitScalarExpr(E->getArg(0)); 4836 // The vector is v2f64, so make sure it's bitcast to that. 4837 Vec = Builder.CreateBitCast(Vec, Ty, "v2f64"); 4838 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); 4839 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); 4840 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 4841 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 4842 // Pairwise addition of a v2f64 into a scalar f64. 4843 return Builder.CreateFAdd(Op0, Op1, "vpaddd"); 4844 } 4845 case NEON::BI__builtin_neon_vpadds_f32: { 4846 llvm::Type *Ty = 4847 llvm::VectorType::get(FloatTy, 2); 4848 Value *Vec = EmitScalarExpr(E->getArg(0)); 4849 // The vector is v2f32, so make sure it's bitcast to that. 4850 Vec = Builder.CreateBitCast(Vec, Ty, "v2f32"); 4851 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); 4852 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); 4853 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 4854 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 4855 // Pairwise addition of a v2f32 into a scalar f32. 4856 return Builder.CreateFAdd(Op0, Op1, "vpaddd"); 4857 } 4858 case NEON::BI__builtin_neon_vceqzd_s64: 4859 case NEON::BI__builtin_neon_vceqzd_f64: 4860 case NEON::BI__builtin_neon_vceqzs_f32: 4861 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4862 return EmitAArch64CompareBuiltinExpr( 4863 Ops[0], ConvertType(E->getCallReturnType(getContext())), 4864 ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ, "vceqz"); 4865 case NEON::BI__builtin_neon_vcgezd_s64: 4866 case NEON::BI__builtin_neon_vcgezd_f64: 4867 case NEON::BI__builtin_neon_vcgezs_f32: 4868 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4869 return EmitAArch64CompareBuiltinExpr( 4870 Ops[0], ConvertType(E->getCallReturnType(getContext())), 4871 ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE, "vcgez"); 4872 case NEON::BI__builtin_neon_vclezd_s64: 4873 case NEON::BI__builtin_neon_vclezd_f64: 4874 case NEON::BI__builtin_neon_vclezs_f32: 4875 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4876 return EmitAArch64CompareBuiltinExpr( 4877 Ops[0], ConvertType(E->getCallReturnType(getContext())), 4878 ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE, "vclez"); 4879 case NEON::BI__builtin_neon_vcgtzd_s64: 4880 case NEON::BI__builtin_neon_vcgtzd_f64: 4881 case NEON::BI__builtin_neon_vcgtzs_f32: 4882 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4883 return EmitAArch64CompareBuiltinExpr( 4884 Ops[0], ConvertType(E->getCallReturnType(getContext())), 4885 ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT, "vcgtz"); 4886 case NEON::BI__builtin_neon_vcltzd_s64: 4887 case NEON::BI__builtin_neon_vcltzd_f64: 4888 case NEON::BI__builtin_neon_vcltzs_f32: 4889 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4890 return EmitAArch64CompareBuiltinExpr( 4891 Ops[0], ConvertType(E->getCallReturnType(getContext())), 4892 ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT, "vcltz"); 4893 4894 case NEON::BI__builtin_neon_vceqzd_u64: { 4895 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4896 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); 4897 Ops[0] = 4898 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty)); 4899 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqzd"); 4900 } 4901 case NEON::BI__builtin_neon_vceqd_f64: 4902 case NEON::BI__builtin_neon_vcled_f64: 4903 case NEON::BI__builtin_neon_vcltd_f64: 4904 case NEON::BI__builtin_neon_vcged_f64: 4905 case NEON::BI__builtin_neon_vcgtd_f64: { 4906 llvm::CmpInst::Predicate P; 4907 switch (BuiltinID) { 4908 default: llvm_unreachable("missing builtin ID in switch!"); 4909 case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ; break; 4910 case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE; break; 4911 case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT; break; 4912 case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE; break; 4913 case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT; break; 4914 } 4915 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4916 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 4917 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); 4918 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); 4919 return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd"); 4920 } 4921 case NEON::BI__builtin_neon_vceqs_f32: 4922 case NEON::BI__builtin_neon_vcles_f32: 4923 case NEON::BI__builtin_neon_vclts_f32: 4924 case NEON::BI__builtin_neon_vcges_f32: 4925 case NEON::BI__builtin_neon_vcgts_f32: { 4926 llvm::CmpInst::Predicate P; 4927 switch (BuiltinID) { 4928 default: llvm_unreachable("missing builtin ID in switch!"); 4929 case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ; break; 4930 case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE; break; 4931 case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT; break; 4932 case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE; break; 4933 case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT; break; 4934 } 4935 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4936 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy); 4937 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy); 4938 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); 4939 return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd"); 4940 } 4941 case NEON::BI__builtin_neon_vceqd_s64: 4942 case NEON::BI__builtin_neon_vceqd_u64: 4943 case NEON::BI__builtin_neon_vcgtd_s64: 4944 case NEON::BI__builtin_neon_vcgtd_u64: 4945 case NEON::BI__builtin_neon_vcltd_s64: 4946 case NEON::BI__builtin_neon_vcltd_u64: 4947 case NEON::BI__builtin_neon_vcged_u64: 4948 case NEON::BI__builtin_neon_vcged_s64: 4949 case NEON::BI__builtin_neon_vcled_u64: 4950 case NEON::BI__builtin_neon_vcled_s64: { 4951 llvm::CmpInst::Predicate P; 4952 switch (BuiltinID) { 4953 default: llvm_unreachable("missing builtin ID in switch!"); 4954 case NEON::BI__builtin_neon_vceqd_s64: 4955 case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;break; 4956 case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;break; 4957 case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;break; 4958 case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;break; 4959 case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;break; 4960 case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;break; 4961 case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;break; 4962 case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;break; 4963 case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;break; 4964 } 4965 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4966 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); 4967 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); 4968 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]); 4969 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd"); 4970 } 4971 case NEON::BI__builtin_neon_vtstd_s64: 4972 case NEON::BI__builtin_neon_vtstd_u64: { 4973 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4974 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); 4975 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); 4976 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); 4977 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], 4978 llvm::Constant::getNullValue(Int64Ty)); 4979 return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd"); 4980 } 4981 case NEON::BI__builtin_neon_vset_lane_i8: 4982 case NEON::BI__builtin_neon_vset_lane_i16: 4983 case NEON::BI__builtin_neon_vset_lane_i32: 4984 case NEON::BI__builtin_neon_vset_lane_i64: 4985 case NEON::BI__builtin_neon_vset_lane_f32: 4986 case NEON::BI__builtin_neon_vsetq_lane_i8: 4987 case NEON::BI__builtin_neon_vsetq_lane_i16: 4988 case NEON::BI__builtin_neon_vsetq_lane_i32: 4989 case NEON::BI__builtin_neon_vsetq_lane_i64: 4990 case NEON::BI__builtin_neon_vsetq_lane_f32: 4991 Ops.push_back(EmitScalarExpr(E->getArg(2))); 4992 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 4993 case NEON::BI__builtin_neon_vset_lane_f64: 4994 // The vector type needs a cast for the v1f64 variant. 4995 Ops[1] = Builder.CreateBitCast(Ops[1], 4996 llvm::VectorType::get(DoubleTy, 1)); 4997 Ops.push_back(EmitScalarExpr(E->getArg(2))); 4998 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 4999 case NEON::BI__builtin_neon_vsetq_lane_f64: 5000 // The vector type needs a cast for the v2f64 variant. 5001 Ops[1] = Builder.CreateBitCast(Ops[1], 5002 llvm::VectorType::get(DoubleTy, 2)); 5003 Ops.push_back(EmitScalarExpr(E->getArg(2))); 5004 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 5005 5006 case NEON::BI__builtin_neon_vget_lane_i8: 5007 case NEON::BI__builtin_neon_vdupb_lane_i8: 5008 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 8)); 5009 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5010 "vget_lane"); 5011 case NEON::BI__builtin_neon_vgetq_lane_i8: 5012 case NEON::BI__builtin_neon_vdupb_laneq_i8: 5013 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 16)); 5014 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5015 "vgetq_lane"); 5016 case NEON::BI__builtin_neon_vget_lane_i16: 5017 case NEON::BI__builtin_neon_vduph_lane_i16: 5018 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 4)); 5019 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5020 "vget_lane"); 5021 case NEON::BI__builtin_neon_vgetq_lane_i16: 5022 case NEON::BI__builtin_neon_vduph_laneq_i16: 5023 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 8)); 5024 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5025 "vgetq_lane"); 5026 case NEON::BI__builtin_neon_vget_lane_i32: 5027 case NEON::BI__builtin_neon_vdups_lane_i32: 5028 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 2)); 5029 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5030 "vget_lane"); 5031 case NEON::BI__builtin_neon_vdups_lane_f32: 5032 Ops[0] = Builder.CreateBitCast(Ops[0], 5033 llvm::VectorType::get(FloatTy, 2)); 5034 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5035 "vdups_lane"); 5036 case NEON::BI__builtin_neon_vgetq_lane_i32: 5037 case NEON::BI__builtin_neon_vdups_laneq_i32: 5038 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4)); 5039 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5040 "vgetq_lane"); 5041 case NEON::BI__builtin_neon_vget_lane_i64: 5042 case NEON::BI__builtin_neon_vdupd_lane_i64: 5043 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 1)); 5044 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5045 "vget_lane"); 5046 case NEON::BI__builtin_neon_vdupd_lane_f64: 5047 Ops[0] = Builder.CreateBitCast(Ops[0], 5048 llvm::VectorType::get(DoubleTy, 1)); 5049 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5050 "vdupd_lane"); 5051 case NEON::BI__builtin_neon_vgetq_lane_i64: 5052 case NEON::BI__builtin_neon_vdupd_laneq_i64: 5053 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2)); 5054 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5055 "vgetq_lane"); 5056 case NEON::BI__builtin_neon_vget_lane_f32: 5057 Ops[0] = Builder.CreateBitCast(Ops[0], 5058 llvm::VectorType::get(FloatTy, 2)); 5059 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5060 "vget_lane"); 5061 case NEON::BI__builtin_neon_vget_lane_f64: 5062 Ops[0] = Builder.CreateBitCast(Ops[0], 5063 llvm::VectorType::get(DoubleTy, 1)); 5064 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5065 "vget_lane"); 5066 case NEON::BI__builtin_neon_vgetq_lane_f32: 5067 case NEON::BI__builtin_neon_vdups_laneq_f32: 5068 Ops[0] = Builder.CreateBitCast(Ops[0], 5069 llvm::VectorType::get(FloatTy, 4)); 5070 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5071 "vgetq_lane"); 5072 case NEON::BI__builtin_neon_vgetq_lane_f64: 5073 case NEON::BI__builtin_neon_vdupd_laneq_f64: 5074 Ops[0] = Builder.CreateBitCast(Ops[0], 5075 llvm::VectorType::get(DoubleTy, 2)); 5076 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5077 "vgetq_lane"); 5078 case NEON::BI__builtin_neon_vaddd_s64: 5079 case NEON::BI__builtin_neon_vaddd_u64: 5080 return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd"); 5081 case NEON::BI__builtin_neon_vsubd_s64: 5082 case NEON::BI__builtin_neon_vsubd_u64: 5083 return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd"); 5084 case NEON::BI__builtin_neon_vqdmlalh_s16: 5085 case NEON::BI__builtin_neon_vqdmlslh_s16: { 5086 SmallVector<Value *, 2> ProductOps; 5087 ProductOps.push_back(vectorWrapScalar16(Ops[1])); 5088 ProductOps.push_back(vectorWrapScalar16(EmitScalarExpr(E->getArg(2)))); 5089 llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4); 5090 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), 5091 ProductOps, "vqdmlXl"); 5092 Constant *CI = ConstantInt::get(SizeTy, 0); 5093 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); 5094 5095 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16 5096 ? Intrinsic::aarch64_neon_sqadd 5097 : Intrinsic::aarch64_neon_sqsub; 5098 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl"); 5099 } 5100 case NEON::BI__builtin_neon_vqshlud_n_s64: { 5101 Ops.push_back(EmitScalarExpr(E->getArg(1))); 5102 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); 5103 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqshlu, Int64Ty), 5104 Ops, "vqshlu_n"); 5105 } 5106 case NEON::BI__builtin_neon_vqshld_n_u64: 5107 case NEON::BI__builtin_neon_vqshld_n_s64: { 5108 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64 5109 ? Intrinsic::aarch64_neon_uqshl 5110 : Intrinsic::aarch64_neon_sqshl; 5111 Ops.push_back(EmitScalarExpr(E->getArg(1))); 5112 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); 5113 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n"); 5114 } 5115 case NEON::BI__builtin_neon_vrshrd_n_u64: 5116 case NEON::BI__builtin_neon_vrshrd_n_s64: { 5117 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64 5118 ? Intrinsic::aarch64_neon_urshl 5119 : Intrinsic::aarch64_neon_srshl; 5120 Ops.push_back(EmitScalarExpr(E->getArg(1))); 5121 int SV = cast<ConstantInt>(Ops[1])->getSExtValue(); 5122 Ops[1] = ConstantInt::get(Int64Ty, -SV); 5123 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n"); 5124 } 5125 case NEON::BI__builtin_neon_vrsrad_n_u64: 5126 case NEON::BI__builtin_neon_vrsrad_n_s64: { 5127 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64 5128 ? Intrinsic::aarch64_neon_urshl 5129 : Intrinsic::aarch64_neon_srshl; 5130 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); 5131 Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2)))); 5132 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty), 5133 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)}); 5134 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty)); 5135 } 5136 case NEON::BI__builtin_neon_vshld_n_s64: 5137 case NEON::BI__builtin_neon_vshld_n_u64: { 5138 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 5139 return Builder.CreateShl( 5140 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n"); 5141 } 5142 case NEON::BI__builtin_neon_vshrd_n_s64: { 5143 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 5144 return Builder.CreateAShr( 5145 Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), 5146 Amt->getZExtValue())), 5147 "shrd_n"); 5148 } 5149 case NEON::BI__builtin_neon_vshrd_n_u64: { 5150 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 5151 uint64_t ShiftAmt = Amt->getZExtValue(); 5152 // Right-shifting an unsigned value by its size yields 0. 5153 if (ShiftAmt == 64) 5154 return ConstantInt::get(Int64Ty, 0); 5155 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt), 5156 "shrd_n"); 5157 } 5158 case NEON::BI__builtin_neon_vsrad_n_s64: { 5159 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2))); 5160 Ops[1] = Builder.CreateAShr( 5161 Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), 5162 Amt->getZExtValue())), 5163 "shrd_n"); 5164 return Builder.CreateAdd(Ops[0], Ops[1]); 5165 } 5166 case NEON::BI__builtin_neon_vsrad_n_u64: { 5167 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2))); 5168 uint64_t ShiftAmt = Amt->getZExtValue(); 5169 // Right-shifting an unsigned value by its size yields 0. 5170 // As Op + 0 = Op, return Ops[0] directly. 5171 if (ShiftAmt == 64) 5172 return Ops[0]; 5173 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt), 5174 "shrd_n"); 5175 return Builder.CreateAdd(Ops[0], Ops[1]); 5176 } 5177 case NEON::BI__builtin_neon_vqdmlalh_lane_s16: 5178 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16: 5179 case NEON::BI__builtin_neon_vqdmlslh_lane_s16: 5180 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: { 5181 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), 5182 "lane"); 5183 SmallVector<Value *, 2> ProductOps; 5184 ProductOps.push_back(vectorWrapScalar16(Ops[1])); 5185 ProductOps.push_back(vectorWrapScalar16(Ops[2])); 5186 llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4); 5187 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), 5188 ProductOps, "vqdmlXl"); 5189 Constant *CI = ConstantInt::get(SizeTy, 0); 5190 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); 5191 Ops.pop_back(); 5192 5193 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 || 5194 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16) 5195 ? Intrinsic::aarch64_neon_sqadd 5196 : Intrinsic::aarch64_neon_sqsub; 5197 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl"); 5198 } 5199 case NEON::BI__builtin_neon_vqdmlals_s32: 5200 case NEON::BI__builtin_neon_vqdmlsls_s32: { 5201 SmallVector<Value *, 2> ProductOps; 5202 ProductOps.push_back(Ops[1]); 5203 ProductOps.push_back(EmitScalarExpr(E->getArg(2))); 5204 Ops[1] = 5205 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar), 5206 ProductOps, "vqdmlXl"); 5207 5208 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32 5209 ? Intrinsic::aarch64_neon_sqadd 5210 : Intrinsic::aarch64_neon_sqsub; 5211 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl"); 5212 } 5213 case NEON::BI__builtin_neon_vqdmlals_lane_s32: 5214 case NEON::BI__builtin_neon_vqdmlals_laneq_s32: 5215 case NEON::BI__builtin_neon_vqdmlsls_lane_s32: 5216 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: { 5217 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), 5218 "lane"); 5219 SmallVector<Value *, 2> ProductOps; 5220 ProductOps.push_back(Ops[1]); 5221 ProductOps.push_back(Ops[2]); 5222 Ops[1] = 5223 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar), 5224 ProductOps, "vqdmlXl"); 5225 Ops.pop_back(); 5226 5227 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 || 5228 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32) 5229 ? Intrinsic::aarch64_neon_sqadd 5230 : Intrinsic::aarch64_neon_sqsub; 5231 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl"); 5232 } 5233 } 5234 5235 llvm::VectorType *VTy = GetNeonType(this, Type); 5236 llvm::Type *Ty = VTy; 5237 if (!Ty) 5238 return nullptr; 5239 5240 // Not all intrinsics handled by the common case work for AArch64 yet, so only 5241 // defer to common code if it's been added to our special map. 5242 Builtin = findNeonIntrinsicInMap(AArch64SIMDIntrinsicMap, BuiltinID, 5243 AArch64SIMDIntrinsicsProvenSorted); 5244 5245 if (Builtin) 5246 return EmitCommonNeonBuiltinExpr( 5247 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic, 5248 Builtin->NameHint, Builtin->TypeModifier, E, Ops, 5249 /*never use addresses*/ Address::invalid(), Address::invalid()); 5250 5251 if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops)) 5252 return V; 5253 5254 unsigned Int; 5255 switch (BuiltinID) { 5256 default: return nullptr; 5257 case NEON::BI__builtin_neon_vbsl_v: 5258 case NEON::BI__builtin_neon_vbslq_v: { 5259 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy); 5260 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl"); 5261 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl"); 5262 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl"); 5263 5264 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl"); 5265 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl"); 5266 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl"); 5267 return Builder.CreateBitCast(Ops[0], Ty); 5268 } 5269 case NEON::BI__builtin_neon_vfma_lane_v: 5270 case NEON::BI__builtin_neon_vfmaq_lane_v: { // Only used for FP types 5271 // The ARM builtins (and instructions) have the addend as the first 5272 // operand, but the 'fma' intrinsics have it last. Swap it around here. 5273 Value *Addend = Ops[0]; 5274 Value *Multiplicand = Ops[1]; 5275 Value *LaneSource = Ops[2]; 5276 Ops[0] = Multiplicand; 5277 Ops[1] = LaneSource; 5278 Ops[2] = Addend; 5279 5280 // Now adjust things to handle the lane access. 5281 llvm::Type *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v ? 5282 llvm::VectorType::get(VTy->getElementType(), VTy->getNumElements() / 2) : 5283 VTy; 5284 llvm::Constant *cst = cast<Constant>(Ops[3]); 5285 Value *SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), cst); 5286 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy); 5287 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane"); 5288 5289 Ops.pop_back(); 5290 Int = Intrinsic::fma; 5291 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla"); 5292 } 5293 case NEON::BI__builtin_neon_vfma_laneq_v: { 5294 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); 5295 // v1f64 fma should be mapped to Neon scalar f64 fma 5296 if (VTy && VTy->getElementType() == DoubleTy) { 5297 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 5298 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); 5299 llvm::Type *VTy = GetNeonType(this, 5300 NeonTypeFlags(NeonTypeFlags::Float64, false, true)); 5301 Ops[2] = Builder.CreateBitCast(Ops[2], VTy); 5302 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); 5303 Value *F = CGM.getIntrinsic(Intrinsic::fma, DoubleTy); 5304 Value *Result = Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); 5305 return Builder.CreateBitCast(Result, Ty); 5306 } 5307 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 5308 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5309 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5310 5311 llvm::Type *STy = llvm::VectorType::get(VTy->getElementType(), 5312 VTy->getNumElements() * 2); 5313 Ops[2] = Builder.CreateBitCast(Ops[2], STy); 5314 Value* SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), 5315 cast<ConstantInt>(Ops[3])); 5316 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane"); 5317 5318 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]}); 5319 } 5320 case NEON::BI__builtin_neon_vfmaq_laneq_v: { 5321 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 5322 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5323 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5324 5325 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 5326 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3])); 5327 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]}); 5328 } 5329 case NEON::BI__builtin_neon_vfmas_lane_f32: 5330 case NEON::BI__builtin_neon_vfmas_laneq_f32: 5331 case NEON::BI__builtin_neon_vfmad_lane_f64: 5332 case NEON::BI__builtin_neon_vfmad_laneq_f64: { 5333 Ops.push_back(EmitScalarExpr(E->getArg(3))); 5334 llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext())); 5335 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 5336 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); 5337 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); 5338 } 5339 case NEON::BI__builtin_neon_vmull_v: 5340 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 5341 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull; 5342 if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull; 5343 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); 5344 case NEON::BI__builtin_neon_vmax_v: 5345 case NEON::BI__builtin_neon_vmaxq_v: 5346 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 5347 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax; 5348 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax; 5349 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax"); 5350 case NEON::BI__builtin_neon_vmin_v: 5351 case NEON::BI__builtin_neon_vminq_v: 5352 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 5353 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin; 5354 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin; 5355 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin"); 5356 case NEON::BI__builtin_neon_vabd_v: 5357 case NEON::BI__builtin_neon_vabdq_v: 5358 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 5359 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd; 5360 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd; 5361 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd"); 5362 case NEON::BI__builtin_neon_vpadal_v: 5363 case NEON::BI__builtin_neon_vpadalq_v: { 5364 unsigned ArgElts = VTy->getNumElements(); 5365 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType()); 5366 unsigned BitWidth = EltTy->getBitWidth(); 5367 llvm::Type *ArgTy = llvm::VectorType::get( 5368 llvm::IntegerType::get(getLLVMContext(), BitWidth/2), 2*ArgElts); 5369 llvm::Type* Tys[2] = { VTy, ArgTy }; 5370 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp; 5371 SmallVector<llvm::Value*, 1> TmpOps; 5372 TmpOps.push_back(Ops[1]); 5373 Function *F = CGM.getIntrinsic(Int, Tys); 5374 llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vpadal"); 5375 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType()); 5376 return Builder.CreateAdd(tmp, addend); 5377 } 5378 case NEON::BI__builtin_neon_vpmin_v: 5379 case NEON::BI__builtin_neon_vpminq_v: 5380 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 5381 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp; 5382 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp; 5383 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin"); 5384 case NEON::BI__builtin_neon_vpmax_v: 5385 case NEON::BI__builtin_neon_vpmaxq_v: 5386 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 5387 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp; 5388 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp; 5389 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax"); 5390 case NEON::BI__builtin_neon_vminnm_v: 5391 case NEON::BI__builtin_neon_vminnmq_v: 5392 Int = Intrinsic::aarch64_neon_fminnm; 5393 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm"); 5394 case NEON::BI__builtin_neon_vmaxnm_v: 5395 case NEON::BI__builtin_neon_vmaxnmq_v: 5396 Int = Intrinsic::aarch64_neon_fmaxnm; 5397 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm"); 5398 case NEON::BI__builtin_neon_vrecpss_f32: { 5399 Ops.push_back(EmitScalarExpr(E->getArg(1))); 5400 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, FloatTy), 5401 Ops, "vrecps"); 5402 } 5403 case NEON::BI__builtin_neon_vrecpsd_f64: { 5404 Ops.push_back(EmitScalarExpr(E->getArg(1))); 5405 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, DoubleTy), 5406 Ops, "vrecps"); 5407 } 5408 case NEON::BI__builtin_neon_vqshrun_n_v: 5409 Int = Intrinsic::aarch64_neon_sqshrun; 5410 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n"); 5411 case NEON::BI__builtin_neon_vqrshrun_n_v: 5412 Int = Intrinsic::aarch64_neon_sqrshrun; 5413 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n"); 5414 case NEON::BI__builtin_neon_vqshrn_n_v: 5415 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn; 5416 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n"); 5417 case NEON::BI__builtin_neon_vrshrn_n_v: 5418 Int = Intrinsic::aarch64_neon_rshrn; 5419 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n"); 5420 case NEON::BI__builtin_neon_vqrshrn_n_v: 5421 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn; 5422 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n"); 5423 case NEON::BI__builtin_neon_vrnda_v: 5424 case NEON::BI__builtin_neon_vrndaq_v: { 5425 Int = Intrinsic::round; 5426 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda"); 5427 } 5428 case NEON::BI__builtin_neon_vrndi_v: 5429 case NEON::BI__builtin_neon_vrndiq_v: { 5430 Int = Intrinsic::nearbyint; 5431 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndi"); 5432 } 5433 case NEON::BI__builtin_neon_vrndm_v: 5434 case NEON::BI__builtin_neon_vrndmq_v: { 5435 Int = Intrinsic::floor; 5436 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm"); 5437 } 5438 case NEON::BI__builtin_neon_vrndn_v: 5439 case NEON::BI__builtin_neon_vrndnq_v: { 5440 Int = Intrinsic::aarch64_neon_frintn; 5441 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn"); 5442 } 5443 case NEON::BI__builtin_neon_vrndp_v: 5444 case NEON::BI__builtin_neon_vrndpq_v: { 5445 Int = Intrinsic::ceil; 5446 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp"); 5447 } 5448 case NEON::BI__builtin_neon_vrndx_v: 5449 case NEON::BI__builtin_neon_vrndxq_v: { 5450 Int = Intrinsic::rint; 5451 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx"); 5452 } 5453 case NEON::BI__builtin_neon_vrnd_v: 5454 case NEON::BI__builtin_neon_vrndq_v: { 5455 Int = Intrinsic::trunc; 5456 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz"); 5457 } 5458 case NEON::BI__builtin_neon_vceqz_v: 5459 case NEON::BI__builtin_neon_vceqzq_v: 5460 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ, 5461 ICmpInst::ICMP_EQ, "vceqz"); 5462 case NEON::BI__builtin_neon_vcgez_v: 5463 case NEON::BI__builtin_neon_vcgezq_v: 5464 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE, 5465 ICmpInst::ICMP_SGE, "vcgez"); 5466 case NEON::BI__builtin_neon_vclez_v: 5467 case NEON::BI__builtin_neon_vclezq_v: 5468 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE, 5469 ICmpInst::ICMP_SLE, "vclez"); 5470 case NEON::BI__builtin_neon_vcgtz_v: 5471 case NEON::BI__builtin_neon_vcgtzq_v: 5472 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT, 5473 ICmpInst::ICMP_SGT, "vcgtz"); 5474 case NEON::BI__builtin_neon_vcltz_v: 5475 case NEON::BI__builtin_neon_vcltzq_v: 5476 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT, 5477 ICmpInst::ICMP_SLT, "vcltz"); 5478 case NEON::BI__builtin_neon_vcvt_f64_v: 5479 case NEON::BI__builtin_neon_vcvtq_f64_v: 5480 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5481 Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad)); 5482 return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") 5483 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); 5484 case NEON::BI__builtin_neon_vcvt_f64_f32: { 5485 assert(Type.getEltType() == NeonTypeFlags::Float64 && quad && 5486 "unexpected vcvt_f64_f32 builtin"); 5487 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float32, false, false); 5488 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); 5489 5490 return Builder.CreateFPExt(Ops[0], Ty, "vcvt"); 5491 } 5492 case NEON::BI__builtin_neon_vcvt_f32_f64: { 5493 assert(Type.getEltType() == NeonTypeFlags::Float32 && 5494 "unexpected vcvt_f32_f64 builtin"); 5495 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float64, false, true); 5496 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); 5497 5498 return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt"); 5499 } 5500 case NEON::BI__builtin_neon_vcvt_s32_v: 5501 case NEON::BI__builtin_neon_vcvt_u32_v: 5502 case NEON::BI__builtin_neon_vcvt_s64_v: 5503 case NEON::BI__builtin_neon_vcvt_u64_v: 5504 case NEON::BI__builtin_neon_vcvtq_s32_v: 5505 case NEON::BI__builtin_neon_vcvtq_u32_v: 5506 case NEON::BI__builtin_neon_vcvtq_s64_v: 5507 case NEON::BI__builtin_neon_vcvtq_u64_v: { 5508 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); 5509 if (usgn) 5510 return Builder.CreateFPToUI(Ops[0], Ty); 5511 return Builder.CreateFPToSI(Ops[0], Ty); 5512 } 5513 case NEON::BI__builtin_neon_vcvta_s32_v: 5514 case NEON::BI__builtin_neon_vcvtaq_s32_v: 5515 case NEON::BI__builtin_neon_vcvta_u32_v: 5516 case NEON::BI__builtin_neon_vcvtaq_u32_v: 5517 case NEON::BI__builtin_neon_vcvta_s64_v: 5518 case NEON::BI__builtin_neon_vcvtaq_s64_v: 5519 case NEON::BI__builtin_neon_vcvta_u64_v: 5520 case NEON::BI__builtin_neon_vcvtaq_u64_v: { 5521 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas; 5522 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 5523 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta"); 5524 } 5525 case NEON::BI__builtin_neon_vcvtm_s32_v: 5526 case NEON::BI__builtin_neon_vcvtmq_s32_v: 5527 case NEON::BI__builtin_neon_vcvtm_u32_v: 5528 case NEON::BI__builtin_neon_vcvtmq_u32_v: 5529 case NEON::BI__builtin_neon_vcvtm_s64_v: 5530 case NEON::BI__builtin_neon_vcvtmq_s64_v: 5531 case NEON::BI__builtin_neon_vcvtm_u64_v: 5532 case NEON::BI__builtin_neon_vcvtmq_u64_v: { 5533 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms; 5534 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 5535 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm"); 5536 } 5537 case NEON::BI__builtin_neon_vcvtn_s32_v: 5538 case NEON::BI__builtin_neon_vcvtnq_s32_v: 5539 case NEON::BI__builtin_neon_vcvtn_u32_v: 5540 case NEON::BI__builtin_neon_vcvtnq_u32_v: 5541 case NEON::BI__builtin_neon_vcvtn_s64_v: 5542 case NEON::BI__builtin_neon_vcvtnq_s64_v: 5543 case NEON::BI__builtin_neon_vcvtn_u64_v: 5544 case NEON::BI__builtin_neon_vcvtnq_u64_v: { 5545 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns; 5546 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 5547 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn"); 5548 } 5549 case NEON::BI__builtin_neon_vcvtp_s32_v: 5550 case NEON::BI__builtin_neon_vcvtpq_s32_v: 5551 case NEON::BI__builtin_neon_vcvtp_u32_v: 5552 case NEON::BI__builtin_neon_vcvtpq_u32_v: 5553 case NEON::BI__builtin_neon_vcvtp_s64_v: 5554 case NEON::BI__builtin_neon_vcvtpq_s64_v: 5555 case NEON::BI__builtin_neon_vcvtp_u64_v: 5556 case NEON::BI__builtin_neon_vcvtpq_u64_v: { 5557 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps; 5558 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 5559 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp"); 5560 } 5561 case NEON::BI__builtin_neon_vmulx_v: 5562 case NEON::BI__builtin_neon_vmulxq_v: { 5563 Int = Intrinsic::aarch64_neon_fmulx; 5564 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx"); 5565 } 5566 case NEON::BI__builtin_neon_vmul_lane_v: 5567 case NEON::BI__builtin_neon_vmul_laneq_v: { 5568 // v1f64 vmul_lane should be mapped to Neon scalar mul lane 5569 bool Quad = false; 5570 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v) 5571 Quad = true; 5572 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 5573 llvm::Type *VTy = GetNeonType(this, 5574 NeonTypeFlags(NeonTypeFlags::Float64, false, Quad)); 5575 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); 5576 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract"); 5577 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]); 5578 return Builder.CreateBitCast(Result, Ty); 5579 } 5580 case NEON::BI__builtin_neon_vnegd_s64: 5581 return Builder.CreateNeg(EmitScalarExpr(E->getArg(0)), "vnegd"); 5582 case NEON::BI__builtin_neon_vpmaxnm_v: 5583 case NEON::BI__builtin_neon_vpmaxnmq_v: { 5584 Int = Intrinsic::aarch64_neon_fmaxnmp; 5585 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm"); 5586 } 5587 case NEON::BI__builtin_neon_vpminnm_v: 5588 case NEON::BI__builtin_neon_vpminnmq_v: { 5589 Int = Intrinsic::aarch64_neon_fminnmp; 5590 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm"); 5591 } 5592 case NEON::BI__builtin_neon_vsqrt_v: 5593 case NEON::BI__builtin_neon_vsqrtq_v: { 5594 Int = Intrinsic::sqrt; 5595 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5596 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt"); 5597 } 5598 case NEON::BI__builtin_neon_vrbit_v: 5599 case NEON::BI__builtin_neon_vrbitq_v: { 5600 Int = Intrinsic::aarch64_neon_rbit; 5601 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit"); 5602 } 5603 case NEON::BI__builtin_neon_vaddv_u8: 5604 // FIXME: These are handled by the AArch64 scalar code. 5605 usgn = true; 5606 // FALLTHROUGH 5607 case NEON::BI__builtin_neon_vaddv_s8: { 5608 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 5609 Ty = Int32Ty; 5610 VTy = llvm::VectorType::get(Int8Ty, 8); 5611 llvm::Type *Tys[2] = { Ty, VTy }; 5612 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5613 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 5614 return Builder.CreateTrunc(Ops[0], Int8Ty); 5615 } 5616 case NEON::BI__builtin_neon_vaddv_u16: 5617 usgn = true; 5618 // FALLTHROUGH 5619 case NEON::BI__builtin_neon_vaddv_s16: { 5620 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 5621 Ty = Int32Ty; 5622 VTy = llvm::VectorType::get(Int16Ty, 4); 5623 llvm::Type *Tys[2] = { Ty, VTy }; 5624 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5625 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 5626 return Builder.CreateTrunc(Ops[0], Int16Ty); 5627 } 5628 case NEON::BI__builtin_neon_vaddvq_u8: 5629 usgn = true; 5630 // FALLTHROUGH 5631 case NEON::BI__builtin_neon_vaddvq_s8: { 5632 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 5633 Ty = Int32Ty; 5634 VTy = llvm::VectorType::get(Int8Ty, 16); 5635 llvm::Type *Tys[2] = { Ty, VTy }; 5636 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5637 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 5638 return Builder.CreateTrunc(Ops[0], Int8Ty); 5639 } 5640 case NEON::BI__builtin_neon_vaddvq_u16: 5641 usgn = true; 5642 // FALLTHROUGH 5643 case NEON::BI__builtin_neon_vaddvq_s16: { 5644 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 5645 Ty = Int32Ty; 5646 VTy = llvm::VectorType::get(Int16Ty, 8); 5647 llvm::Type *Tys[2] = { Ty, VTy }; 5648 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5649 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 5650 return Builder.CreateTrunc(Ops[0], Int16Ty); 5651 } 5652 case NEON::BI__builtin_neon_vmaxv_u8: { 5653 Int = Intrinsic::aarch64_neon_umaxv; 5654 Ty = Int32Ty; 5655 VTy = llvm::VectorType::get(Int8Ty, 8); 5656 llvm::Type *Tys[2] = { Ty, VTy }; 5657 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5658 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5659 return Builder.CreateTrunc(Ops[0], Int8Ty); 5660 } 5661 case NEON::BI__builtin_neon_vmaxv_u16: { 5662 Int = Intrinsic::aarch64_neon_umaxv; 5663 Ty = Int32Ty; 5664 VTy = llvm::VectorType::get(Int16Ty, 4); 5665 llvm::Type *Tys[2] = { Ty, VTy }; 5666 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5667 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5668 return Builder.CreateTrunc(Ops[0], Int16Ty); 5669 } 5670 case NEON::BI__builtin_neon_vmaxvq_u8: { 5671 Int = Intrinsic::aarch64_neon_umaxv; 5672 Ty = Int32Ty; 5673 VTy = llvm::VectorType::get(Int8Ty, 16); 5674 llvm::Type *Tys[2] = { Ty, VTy }; 5675 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5676 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5677 return Builder.CreateTrunc(Ops[0], Int8Ty); 5678 } 5679 case NEON::BI__builtin_neon_vmaxvq_u16: { 5680 Int = Intrinsic::aarch64_neon_umaxv; 5681 Ty = Int32Ty; 5682 VTy = llvm::VectorType::get(Int16Ty, 8); 5683 llvm::Type *Tys[2] = { Ty, VTy }; 5684 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5685 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5686 return Builder.CreateTrunc(Ops[0], Int16Ty); 5687 } 5688 case NEON::BI__builtin_neon_vmaxv_s8: { 5689 Int = Intrinsic::aarch64_neon_smaxv; 5690 Ty = Int32Ty; 5691 VTy = llvm::VectorType::get(Int8Ty, 8); 5692 llvm::Type *Tys[2] = { Ty, VTy }; 5693 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5694 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5695 return Builder.CreateTrunc(Ops[0], Int8Ty); 5696 } 5697 case NEON::BI__builtin_neon_vmaxv_s16: { 5698 Int = Intrinsic::aarch64_neon_smaxv; 5699 Ty = Int32Ty; 5700 VTy = llvm::VectorType::get(Int16Ty, 4); 5701 llvm::Type *Tys[2] = { Ty, VTy }; 5702 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5703 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5704 return Builder.CreateTrunc(Ops[0], Int16Ty); 5705 } 5706 case NEON::BI__builtin_neon_vmaxvq_s8: { 5707 Int = Intrinsic::aarch64_neon_smaxv; 5708 Ty = Int32Ty; 5709 VTy = llvm::VectorType::get(Int8Ty, 16); 5710 llvm::Type *Tys[2] = { Ty, VTy }; 5711 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5712 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5713 return Builder.CreateTrunc(Ops[0], Int8Ty); 5714 } 5715 case NEON::BI__builtin_neon_vmaxvq_s16: { 5716 Int = Intrinsic::aarch64_neon_smaxv; 5717 Ty = Int32Ty; 5718 VTy = llvm::VectorType::get(Int16Ty, 8); 5719 llvm::Type *Tys[2] = { Ty, VTy }; 5720 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5721 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5722 return Builder.CreateTrunc(Ops[0], Int16Ty); 5723 } 5724 case NEON::BI__builtin_neon_vminv_u8: { 5725 Int = Intrinsic::aarch64_neon_uminv; 5726 Ty = Int32Ty; 5727 VTy = llvm::VectorType::get(Int8Ty, 8); 5728 llvm::Type *Tys[2] = { Ty, VTy }; 5729 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5730 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5731 return Builder.CreateTrunc(Ops[0], Int8Ty); 5732 } 5733 case NEON::BI__builtin_neon_vminv_u16: { 5734 Int = Intrinsic::aarch64_neon_uminv; 5735 Ty = Int32Ty; 5736 VTy = llvm::VectorType::get(Int16Ty, 4); 5737 llvm::Type *Tys[2] = { Ty, VTy }; 5738 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5739 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5740 return Builder.CreateTrunc(Ops[0], Int16Ty); 5741 } 5742 case NEON::BI__builtin_neon_vminvq_u8: { 5743 Int = Intrinsic::aarch64_neon_uminv; 5744 Ty = Int32Ty; 5745 VTy = llvm::VectorType::get(Int8Ty, 16); 5746 llvm::Type *Tys[2] = { Ty, VTy }; 5747 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5748 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5749 return Builder.CreateTrunc(Ops[0], Int8Ty); 5750 } 5751 case NEON::BI__builtin_neon_vminvq_u16: { 5752 Int = Intrinsic::aarch64_neon_uminv; 5753 Ty = Int32Ty; 5754 VTy = llvm::VectorType::get(Int16Ty, 8); 5755 llvm::Type *Tys[2] = { Ty, VTy }; 5756 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5757 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5758 return Builder.CreateTrunc(Ops[0], Int16Ty); 5759 } 5760 case NEON::BI__builtin_neon_vminv_s8: { 5761 Int = Intrinsic::aarch64_neon_sminv; 5762 Ty = Int32Ty; 5763 VTy = llvm::VectorType::get(Int8Ty, 8); 5764 llvm::Type *Tys[2] = { Ty, VTy }; 5765 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5766 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5767 return Builder.CreateTrunc(Ops[0], Int8Ty); 5768 } 5769 case NEON::BI__builtin_neon_vminv_s16: { 5770 Int = Intrinsic::aarch64_neon_sminv; 5771 Ty = Int32Ty; 5772 VTy = llvm::VectorType::get(Int16Ty, 4); 5773 llvm::Type *Tys[2] = { Ty, VTy }; 5774 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5775 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5776 return Builder.CreateTrunc(Ops[0], Int16Ty); 5777 } 5778 case NEON::BI__builtin_neon_vminvq_s8: { 5779 Int = Intrinsic::aarch64_neon_sminv; 5780 Ty = Int32Ty; 5781 VTy = llvm::VectorType::get(Int8Ty, 16); 5782 llvm::Type *Tys[2] = { Ty, VTy }; 5783 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5784 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5785 return Builder.CreateTrunc(Ops[0], Int8Ty); 5786 } 5787 case NEON::BI__builtin_neon_vminvq_s16: { 5788 Int = Intrinsic::aarch64_neon_sminv; 5789 Ty = Int32Ty; 5790 VTy = llvm::VectorType::get(Int16Ty, 8); 5791 llvm::Type *Tys[2] = { Ty, VTy }; 5792 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5793 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5794 return Builder.CreateTrunc(Ops[0], Int16Ty); 5795 } 5796 case NEON::BI__builtin_neon_vmul_n_f64: { 5797 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 5798 Value *RHS = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), DoubleTy); 5799 return Builder.CreateFMul(Ops[0], RHS); 5800 } 5801 case NEON::BI__builtin_neon_vaddlv_u8: { 5802 Int = Intrinsic::aarch64_neon_uaddlv; 5803 Ty = Int32Ty; 5804 VTy = llvm::VectorType::get(Int8Ty, 8); 5805 llvm::Type *Tys[2] = { Ty, VTy }; 5806 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5807 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5808 return Builder.CreateTrunc(Ops[0], Int16Ty); 5809 } 5810 case NEON::BI__builtin_neon_vaddlv_u16: { 5811 Int = Intrinsic::aarch64_neon_uaddlv; 5812 Ty = Int32Ty; 5813 VTy = llvm::VectorType::get(Int16Ty, 4); 5814 llvm::Type *Tys[2] = { Ty, VTy }; 5815 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5816 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5817 } 5818 case NEON::BI__builtin_neon_vaddlvq_u8: { 5819 Int = Intrinsic::aarch64_neon_uaddlv; 5820 Ty = Int32Ty; 5821 VTy = llvm::VectorType::get(Int8Ty, 16); 5822 llvm::Type *Tys[2] = { Ty, VTy }; 5823 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5824 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5825 return Builder.CreateTrunc(Ops[0], Int16Ty); 5826 } 5827 case NEON::BI__builtin_neon_vaddlvq_u16: { 5828 Int = Intrinsic::aarch64_neon_uaddlv; 5829 Ty = Int32Ty; 5830 VTy = llvm::VectorType::get(Int16Ty, 8); 5831 llvm::Type *Tys[2] = { Ty, VTy }; 5832 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5833 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5834 } 5835 case NEON::BI__builtin_neon_vaddlv_s8: { 5836 Int = Intrinsic::aarch64_neon_saddlv; 5837 Ty = Int32Ty; 5838 VTy = llvm::VectorType::get(Int8Ty, 8); 5839 llvm::Type *Tys[2] = { Ty, VTy }; 5840 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5841 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5842 return Builder.CreateTrunc(Ops[0], Int16Ty); 5843 } 5844 case NEON::BI__builtin_neon_vaddlv_s16: { 5845 Int = Intrinsic::aarch64_neon_saddlv; 5846 Ty = Int32Ty; 5847 VTy = llvm::VectorType::get(Int16Ty, 4); 5848 llvm::Type *Tys[2] = { Ty, VTy }; 5849 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5850 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5851 } 5852 case NEON::BI__builtin_neon_vaddlvq_s8: { 5853 Int = Intrinsic::aarch64_neon_saddlv; 5854 Ty = Int32Ty; 5855 VTy = llvm::VectorType::get(Int8Ty, 16); 5856 llvm::Type *Tys[2] = { Ty, VTy }; 5857 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5858 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5859 return Builder.CreateTrunc(Ops[0], Int16Ty); 5860 } 5861 case NEON::BI__builtin_neon_vaddlvq_s16: { 5862 Int = Intrinsic::aarch64_neon_saddlv; 5863 Ty = Int32Ty; 5864 VTy = llvm::VectorType::get(Int16Ty, 8); 5865 llvm::Type *Tys[2] = { Ty, VTy }; 5866 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5867 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5868 } 5869 case NEON::BI__builtin_neon_vsri_n_v: 5870 case NEON::BI__builtin_neon_vsriq_n_v: { 5871 Int = Intrinsic::aarch64_neon_vsri; 5872 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty); 5873 return EmitNeonCall(Intrin, Ops, "vsri_n"); 5874 } 5875 case NEON::BI__builtin_neon_vsli_n_v: 5876 case NEON::BI__builtin_neon_vsliq_n_v: { 5877 Int = Intrinsic::aarch64_neon_vsli; 5878 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty); 5879 return EmitNeonCall(Intrin, Ops, "vsli_n"); 5880 } 5881 case NEON::BI__builtin_neon_vsra_n_v: 5882 case NEON::BI__builtin_neon_vsraq_n_v: 5883 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5884 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); 5885 return Builder.CreateAdd(Ops[0], Ops[1]); 5886 case NEON::BI__builtin_neon_vrsra_n_v: 5887 case NEON::BI__builtin_neon_vrsraq_n_v: { 5888 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl; 5889 SmallVector<llvm::Value*,2> TmpOps; 5890 TmpOps.push_back(Ops[1]); 5891 TmpOps.push_back(Ops[2]); 5892 Function* F = CGM.getIntrinsic(Int, Ty); 5893 llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vrshr_n", 1, true); 5894 Ops[0] = Builder.CreateBitCast(Ops[0], VTy); 5895 return Builder.CreateAdd(Ops[0], tmp); 5896 } 5897 // FIXME: Sharing loads & stores with 32-bit is complicated by the absence 5898 // of an Align parameter here. 5899 case NEON::BI__builtin_neon_vld1_x2_v: 5900 case NEON::BI__builtin_neon_vld1q_x2_v: 5901 case NEON::BI__builtin_neon_vld1_x3_v: 5902 case NEON::BI__builtin_neon_vld1q_x3_v: 5903 case NEON::BI__builtin_neon_vld1_x4_v: 5904 case NEON::BI__builtin_neon_vld1q_x4_v: { 5905 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType()); 5906 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5907 llvm::Type *Tys[2] = { VTy, PTy }; 5908 unsigned Int; 5909 switch (BuiltinID) { 5910 case NEON::BI__builtin_neon_vld1_x2_v: 5911 case NEON::BI__builtin_neon_vld1q_x2_v: 5912 Int = Intrinsic::aarch64_neon_ld1x2; 5913 break; 5914 case NEON::BI__builtin_neon_vld1_x3_v: 5915 case NEON::BI__builtin_neon_vld1q_x3_v: 5916 Int = Intrinsic::aarch64_neon_ld1x3; 5917 break; 5918 case NEON::BI__builtin_neon_vld1_x4_v: 5919 case NEON::BI__builtin_neon_vld1q_x4_v: 5920 Int = Intrinsic::aarch64_neon_ld1x4; 5921 break; 5922 } 5923 Function *F = CGM.getIntrinsic(Int, Tys); 5924 Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN"); 5925 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 5926 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5927 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 5928 } 5929 case NEON::BI__builtin_neon_vst1_x2_v: 5930 case NEON::BI__builtin_neon_vst1q_x2_v: 5931 case NEON::BI__builtin_neon_vst1_x3_v: 5932 case NEON::BI__builtin_neon_vst1q_x3_v: 5933 case NEON::BI__builtin_neon_vst1_x4_v: 5934 case NEON::BI__builtin_neon_vst1q_x4_v: { 5935 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType()); 5936 llvm::Type *Tys[2] = { VTy, PTy }; 5937 unsigned Int; 5938 switch (BuiltinID) { 5939 case NEON::BI__builtin_neon_vst1_x2_v: 5940 case NEON::BI__builtin_neon_vst1q_x2_v: 5941 Int = Intrinsic::aarch64_neon_st1x2; 5942 break; 5943 case NEON::BI__builtin_neon_vst1_x3_v: 5944 case NEON::BI__builtin_neon_vst1q_x3_v: 5945 Int = Intrinsic::aarch64_neon_st1x3; 5946 break; 5947 case NEON::BI__builtin_neon_vst1_x4_v: 5948 case NEON::BI__builtin_neon_vst1q_x4_v: 5949 Int = Intrinsic::aarch64_neon_st1x4; 5950 break; 5951 } 5952 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); 5953 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, ""); 5954 } 5955 case NEON::BI__builtin_neon_vld1_v: 5956 case NEON::BI__builtin_neon_vld1q_v: 5957 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); 5958 return Builder.CreateDefaultAlignedLoad(Ops[0]); 5959 case NEON::BI__builtin_neon_vst1_v: 5960 case NEON::BI__builtin_neon_vst1q_v: 5961 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); 5962 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); 5963 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 5964 case NEON::BI__builtin_neon_vld1_lane_v: 5965 case NEON::BI__builtin_neon_vld1q_lane_v: 5966 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5967 Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 5968 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5969 Ops[0] = Builder.CreateDefaultAlignedLoad(Ops[0]); 5970 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane"); 5971 case NEON::BI__builtin_neon_vld1_dup_v: 5972 case NEON::BI__builtin_neon_vld1q_dup_v: { 5973 Value *V = UndefValue::get(Ty); 5974 Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 5975 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5976 Ops[0] = Builder.CreateDefaultAlignedLoad(Ops[0]); 5977 llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); 5978 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI); 5979 return EmitNeonSplat(Ops[0], CI); 5980 } 5981 case NEON::BI__builtin_neon_vst1_lane_v: 5982 case NEON::BI__builtin_neon_vst1q_lane_v: 5983 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5984 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); 5985 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 5986 return Builder.CreateDefaultAlignedStore(Ops[1], 5987 Builder.CreateBitCast(Ops[0], Ty)); 5988 case NEON::BI__builtin_neon_vld2_v: 5989 case NEON::BI__builtin_neon_vld2q_v: { 5990 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 5991 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5992 llvm::Type *Tys[2] = { VTy, PTy }; 5993 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys); 5994 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); 5995 Ops[0] = Builder.CreateBitCast(Ops[0], 5996 llvm::PointerType::getUnqual(Ops[1]->getType())); 5997 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 5998 } 5999 case NEON::BI__builtin_neon_vld3_v: 6000 case NEON::BI__builtin_neon_vld3q_v: { 6001 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 6002 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6003 llvm::Type *Tys[2] = { VTy, PTy }; 6004 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys); 6005 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); 6006 Ops[0] = Builder.CreateBitCast(Ops[0], 6007 llvm::PointerType::getUnqual(Ops[1]->getType())); 6008 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6009 } 6010 case NEON::BI__builtin_neon_vld4_v: 6011 case NEON::BI__builtin_neon_vld4q_v: { 6012 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 6013 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6014 llvm::Type *Tys[2] = { VTy, PTy }; 6015 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys); 6016 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); 6017 Ops[0] = Builder.CreateBitCast(Ops[0], 6018 llvm::PointerType::getUnqual(Ops[1]->getType())); 6019 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6020 } 6021 case NEON::BI__builtin_neon_vld2_dup_v: 6022 case NEON::BI__builtin_neon_vld2q_dup_v: { 6023 llvm::Type *PTy = 6024 llvm::PointerType::getUnqual(VTy->getElementType()); 6025 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6026 llvm::Type *Tys[2] = { VTy, PTy }; 6027 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys); 6028 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); 6029 Ops[0] = Builder.CreateBitCast(Ops[0], 6030 llvm::PointerType::getUnqual(Ops[1]->getType())); 6031 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6032 } 6033 case NEON::BI__builtin_neon_vld3_dup_v: 6034 case NEON::BI__builtin_neon_vld3q_dup_v: { 6035 llvm::Type *PTy = 6036 llvm::PointerType::getUnqual(VTy->getElementType()); 6037 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6038 llvm::Type *Tys[2] = { VTy, PTy }; 6039 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys); 6040 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); 6041 Ops[0] = Builder.CreateBitCast(Ops[0], 6042 llvm::PointerType::getUnqual(Ops[1]->getType())); 6043 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6044 } 6045 case NEON::BI__builtin_neon_vld4_dup_v: 6046 case NEON::BI__builtin_neon_vld4q_dup_v: { 6047 llvm::Type *PTy = 6048 llvm::PointerType::getUnqual(VTy->getElementType()); 6049 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6050 llvm::Type *Tys[2] = { VTy, PTy }; 6051 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys); 6052 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); 6053 Ops[0] = Builder.CreateBitCast(Ops[0], 6054 llvm::PointerType::getUnqual(Ops[1]->getType())); 6055 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6056 } 6057 case NEON::BI__builtin_neon_vld2_lane_v: 6058 case NEON::BI__builtin_neon_vld2q_lane_v: { 6059 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 6060 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys); 6061 Ops.push_back(Ops[1]); 6062 Ops.erase(Ops.begin()+1); 6063 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 6064 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 6065 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); 6066 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane"); 6067 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 6068 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 6069 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6070 } 6071 case NEON::BI__builtin_neon_vld3_lane_v: 6072 case NEON::BI__builtin_neon_vld3q_lane_v: { 6073 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 6074 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys); 6075 Ops.push_back(Ops[1]); 6076 Ops.erase(Ops.begin()+1); 6077 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 6078 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 6079 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); 6080 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); 6081 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane"); 6082 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 6083 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 6084 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6085 } 6086 case NEON::BI__builtin_neon_vld4_lane_v: 6087 case NEON::BI__builtin_neon_vld4q_lane_v: { 6088 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 6089 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys); 6090 Ops.push_back(Ops[1]); 6091 Ops.erase(Ops.begin()+1); 6092 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 6093 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 6094 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); 6095 Ops[4] = Builder.CreateBitCast(Ops[4], Ty); 6096 Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty); 6097 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld4_lane"); 6098 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 6099 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 6100 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6101 } 6102 case NEON::BI__builtin_neon_vst2_v: 6103 case NEON::BI__builtin_neon_vst2q_v: { 6104 Ops.push_back(Ops[0]); 6105 Ops.erase(Ops.begin()); 6106 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() }; 6107 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys), 6108 Ops, ""); 6109 } 6110 case NEON::BI__builtin_neon_vst2_lane_v: 6111 case NEON::BI__builtin_neon_vst2q_lane_v: { 6112 Ops.push_back(Ops[0]); 6113 Ops.erase(Ops.begin()); 6114 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty); 6115 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; 6116 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys), 6117 Ops, ""); 6118 } 6119 case NEON::BI__builtin_neon_vst3_v: 6120 case NEON::BI__builtin_neon_vst3q_v: { 6121 Ops.push_back(Ops[0]); 6122 Ops.erase(Ops.begin()); 6123 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; 6124 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys), 6125 Ops, ""); 6126 } 6127 case NEON::BI__builtin_neon_vst3_lane_v: 6128 case NEON::BI__builtin_neon_vst3q_lane_v: { 6129 Ops.push_back(Ops[0]); 6130 Ops.erase(Ops.begin()); 6131 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); 6132 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; 6133 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys), 6134 Ops, ""); 6135 } 6136 case NEON::BI__builtin_neon_vst4_v: 6137 case NEON::BI__builtin_neon_vst4q_v: { 6138 Ops.push_back(Ops[0]); 6139 Ops.erase(Ops.begin()); 6140 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; 6141 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys), 6142 Ops, ""); 6143 } 6144 case NEON::BI__builtin_neon_vst4_lane_v: 6145 case NEON::BI__builtin_neon_vst4q_lane_v: { 6146 Ops.push_back(Ops[0]); 6147 Ops.erase(Ops.begin()); 6148 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); 6149 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() }; 6150 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys), 6151 Ops, ""); 6152 } 6153 case NEON::BI__builtin_neon_vtrn_v: 6154 case NEON::BI__builtin_neon_vtrnq_v: { 6155 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 6156 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 6157 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 6158 Value *SV = nullptr; 6159 6160 for (unsigned vi = 0; vi != 2; ++vi) { 6161 SmallVector<int, 16> Indices; 6162 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 6163 Indices.push_back(i+vi); 6164 Indices.push_back(i+e+vi); 6165 } 6166 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 6167 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn"); 6168 SV = Builder.CreateDefaultAlignedStore(SV, Addr); 6169 } 6170 return SV; 6171 } 6172 case NEON::BI__builtin_neon_vuzp_v: 6173 case NEON::BI__builtin_neon_vuzpq_v: { 6174 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 6175 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 6176 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 6177 Value *SV = nullptr; 6178 6179 for (unsigned vi = 0; vi != 2; ++vi) { 6180 SmallVector<int, 16> Indices; 6181 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 6182 Indices.push_back(2*i+vi); 6183 6184 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 6185 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp"); 6186 SV = Builder.CreateDefaultAlignedStore(SV, Addr); 6187 } 6188 return SV; 6189 } 6190 case NEON::BI__builtin_neon_vzip_v: 6191 case NEON::BI__builtin_neon_vzipq_v: { 6192 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 6193 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 6194 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 6195 Value *SV = nullptr; 6196 6197 for (unsigned vi = 0; vi != 2; ++vi) { 6198 SmallVector<int, 16> Indices; 6199 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 6200 Indices.push_back((i + vi*e) >> 1); 6201 Indices.push_back(((i + vi*e) >> 1)+e); 6202 } 6203 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 6204 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip"); 6205 SV = Builder.CreateDefaultAlignedStore(SV, Addr); 6206 } 6207 return SV; 6208 } 6209 case NEON::BI__builtin_neon_vqtbl1q_v: { 6210 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty), 6211 Ops, "vtbl1"); 6212 } 6213 case NEON::BI__builtin_neon_vqtbl2q_v: { 6214 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty), 6215 Ops, "vtbl2"); 6216 } 6217 case NEON::BI__builtin_neon_vqtbl3q_v: { 6218 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty), 6219 Ops, "vtbl3"); 6220 } 6221 case NEON::BI__builtin_neon_vqtbl4q_v: { 6222 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty), 6223 Ops, "vtbl4"); 6224 } 6225 case NEON::BI__builtin_neon_vqtbx1q_v: { 6226 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty), 6227 Ops, "vtbx1"); 6228 } 6229 case NEON::BI__builtin_neon_vqtbx2q_v: { 6230 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty), 6231 Ops, "vtbx2"); 6232 } 6233 case NEON::BI__builtin_neon_vqtbx3q_v: { 6234 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty), 6235 Ops, "vtbx3"); 6236 } 6237 case NEON::BI__builtin_neon_vqtbx4q_v: { 6238 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty), 6239 Ops, "vtbx4"); 6240 } 6241 case NEON::BI__builtin_neon_vsqadd_v: 6242 case NEON::BI__builtin_neon_vsqaddq_v: { 6243 Int = Intrinsic::aarch64_neon_usqadd; 6244 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd"); 6245 } 6246 case NEON::BI__builtin_neon_vuqadd_v: 6247 case NEON::BI__builtin_neon_vuqaddq_v: { 6248 Int = Intrinsic::aarch64_neon_suqadd; 6249 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd"); 6250 } 6251 } 6252 } 6253 6254 llvm::Value *CodeGenFunction:: 6255 BuildVector(ArrayRef<llvm::Value*> Ops) { 6256 assert((Ops.size() & (Ops.size() - 1)) == 0 && 6257 "Not a power-of-two sized vector!"); 6258 bool AllConstants = true; 6259 for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i) 6260 AllConstants &= isa<Constant>(Ops[i]); 6261 6262 // If this is a constant vector, create a ConstantVector. 6263 if (AllConstants) { 6264 SmallVector<llvm::Constant*, 16> CstOps; 6265 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 6266 CstOps.push_back(cast<Constant>(Ops[i])); 6267 return llvm::ConstantVector::get(CstOps); 6268 } 6269 6270 // Otherwise, insertelement the values to build the vector. 6271 Value *Result = 6272 llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size())); 6273 6274 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 6275 Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i)); 6276 6277 return Result; 6278 } 6279 6280 static Value *EmitX86MaskedStore(CodeGenFunction &CGF, 6281 SmallVectorImpl<Value *> &Ops, 6282 unsigned Align) { 6283 // Cast the pointer to right type. 6284 Ops[0] = CGF.Builder.CreateBitCast(Ops[0], 6285 llvm::PointerType::getUnqual(Ops[1]->getType())); 6286 6287 // If the mask is all ones just emit a regular store. 6288 if (const auto *C = dyn_cast<Constant>(Ops[2])) 6289 if (C->isAllOnesValue()) 6290 return CGF.Builder.CreateAlignedStore(Ops[1], Ops[0], Align); 6291 6292 // Convert the mask from an integer type to a vector of i1. 6293 unsigned NumElts = Ops[1]->getType()->getVectorNumElements(); 6294 llvm::VectorType *MaskTy = llvm::VectorType::get(CGF.Builder.getInt1Ty(), 6295 cast<IntegerType>(Ops[2]->getType())->getBitWidth()); 6296 Ops[2] = CGF.Builder.CreateBitCast(Ops[2], MaskTy); 6297 6298 // If we have less than 8 elements, then the starting mask was an i8 and 6299 // we need to extract down to the right number of elements. 6300 if (NumElts < 8) { 6301 int Indices[4]; 6302 for (unsigned i = 0; i != NumElts; ++i) 6303 Indices[i] = i; 6304 Ops[2] = CGF.Builder.CreateShuffleVector(Ops[2], Ops[2], 6305 makeArrayRef(Indices, NumElts), 6306 "extract"); 6307 } 6308 6309 return CGF.Builder.CreateMaskedStore(Ops[1], Ops[0], Align, Ops[2]); 6310 } 6311 6312 static Value *EmitX86MaskedLoad(CodeGenFunction &CGF, 6313 SmallVectorImpl<Value *> &Ops, unsigned Align) { 6314 // Cast the pointer to right type. 6315 Ops[0] = CGF.Builder.CreateBitCast(Ops[0], 6316 llvm::PointerType::getUnqual(Ops[1]->getType())); 6317 6318 // If the mask is all ones just emit a regular store. 6319 if (const auto *C = dyn_cast<Constant>(Ops[2])) 6320 if (C->isAllOnesValue()) 6321 return CGF.Builder.CreateAlignedLoad(Ops[0], Align); 6322 6323 // Convert the mask from an integer type to a vector of i1. 6324 unsigned NumElts = Ops[1]->getType()->getVectorNumElements(); 6325 llvm::VectorType *MaskTy = llvm::VectorType::get(CGF.Builder.getInt1Ty(), 6326 cast<IntegerType>(Ops[2]->getType())->getBitWidth()); 6327 Ops[2] = CGF.Builder.CreateBitCast(Ops[2], MaskTy); 6328 6329 // If we have less than 8 elements, then the starting mask was an i8 and 6330 // we need to extract down to the right number of elements. 6331 if (NumElts < 8) { 6332 int Indices[4]; 6333 for (unsigned i = 0; i != NumElts; ++i) 6334 Indices[i] = i; 6335 Ops[2] = CGF.Builder.CreateShuffleVector(Ops[2], Ops[2], 6336 makeArrayRef(Indices, NumElts), 6337 "extract"); 6338 } 6339 6340 return CGF.Builder.CreateMaskedLoad(Ops[0], Align, Ops[2], Ops[1]); 6341 } 6342 6343 Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, 6344 const CallExpr *E) { 6345 if (BuiltinID == X86::BI__builtin_ms_va_start || 6346 BuiltinID == X86::BI__builtin_ms_va_end) 6347 return EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).getPointer(), 6348 BuiltinID == X86::BI__builtin_ms_va_start); 6349 if (BuiltinID == X86::BI__builtin_ms_va_copy) { 6350 // Lower this manually. We can't reliably determine whether or not any 6351 // given va_copy() is for a Win64 va_list from the calling convention 6352 // alone, because it's legal to do this from a System V ABI function. 6353 // With opaque pointer types, we won't have enough information in LLVM 6354 // IR to determine this from the argument types, either. Best to do it 6355 // now, while we have enough information. 6356 Address DestAddr = EmitMSVAListRef(E->getArg(0)); 6357 Address SrcAddr = EmitMSVAListRef(E->getArg(1)); 6358 6359 llvm::Type *BPP = Int8PtrPtrTy; 6360 6361 DestAddr = Address(Builder.CreateBitCast(DestAddr.getPointer(), BPP, "cp"), 6362 DestAddr.getAlignment()); 6363 SrcAddr = Address(Builder.CreateBitCast(SrcAddr.getPointer(), BPP, "ap"), 6364 SrcAddr.getAlignment()); 6365 6366 Value *ArgPtr = Builder.CreateLoad(SrcAddr, "ap.val"); 6367 return Builder.CreateStore(ArgPtr, DestAddr); 6368 } 6369 6370 SmallVector<Value*, 4> Ops; 6371 6372 // Find out if any arguments are required to be integer constant expressions. 6373 unsigned ICEArguments = 0; 6374 ASTContext::GetBuiltinTypeError Error; 6375 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 6376 assert(Error == ASTContext::GE_None && "Should not codegen an error"); 6377 6378 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) { 6379 // If this is a normal argument, just emit it as a scalar. 6380 if ((ICEArguments & (1 << i)) == 0) { 6381 Ops.push_back(EmitScalarExpr(E->getArg(i))); 6382 continue; 6383 } 6384 6385 // If this is required to be a constant, constant fold it so that we know 6386 // that the generated intrinsic gets a ConstantInt. 6387 llvm::APSInt Result; 6388 bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext()); 6389 assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst; 6390 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); 6391 } 6392 6393 switch (BuiltinID) { 6394 default: return nullptr; 6395 case X86::BI__builtin_cpu_supports: { 6396 const Expr *FeatureExpr = E->getArg(0)->IgnoreParenCasts(); 6397 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString(); 6398 6399 // TODO: When/if this becomes more than x86 specific then use a TargetInfo 6400 // based mapping. 6401 // Processor features and mapping to processor feature value. 6402 enum X86Features { 6403 CMOV = 0, 6404 MMX, 6405 POPCNT, 6406 SSE, 6407 SSE2, 6408 SSE3, 6409 SSSE3, 6410 SSE4_1, 6411 SSE4_2, 6412 AVX, 6413 AVX2, 6414 SSE4_A, 6415 FMA4, 6416 XOP, 6417 FMA, 6418 AVX512F, 6419 BMI, 6420 BMI2, 6421 AES, 6422 PCLMUL, 6423 AVX512VL, 6424 AVX512BW, 6425 AVX512DQ, 6426 AVX512CD, 6427 AVX512ER, 6428 AVX512PF, 6429 AVX512VBMI, 6430 AVX512IFMA, 6431 MAX 6432 }; 6433 6434 X86Features Feature = StringSwitch<X86Features>(FeatureStr) 6435 .Case("cmov", X86Features::CMOV) 6436 .Case("mmx", X86Features::MMX) 6437 .Case("popcnt", X86Features::POPCNT) 6438 .Case("sse", X86Features::SSE) 6439 .Case("sse2", X86Features::SSE2) 6440 .Case("sse3", X86Features::SSE3) 6441 .Case("ssse3", X86Features::SSSE3) 6442 .Case("sse4.1", X86Features::SSE4_1) 6443 .Case("sse4.2", X86Features::SSE4_2) 6444 .Case("avx", X86Features::AVX) 6445 .Case("avx2", X86Features::AVX2) 6446 .Case("sse4a", X86Features::SSE4_A) 6447 .Case("fma4", X86Features::FMA4) 6448 .Case("xop", X86Features::XOP) 6449 .Case("fma", X86Features::FMA) 6450 .Case("avx512f", X86Features::AVX512F) 6451 .Case("bmi", X86Features::BMI) 6452 .Case("bmi2", X86Features::BMI2) 6453 .Case("aes", X86Features::AES) 6454 .Case("pclmul", X86Features::PCLMUL) 6455 .Case("avx512vl", X86Features::AVX512VL) 6456 .Case("avx512bw", X86Features::AVX512BW) 6457 .Case("avx512dq", X86Features::AVX512DQ) 6458 .Case("avx512cd", X86Features::AVX512CD) 6459 .Case("avx512er", X86Features::AVX512ER) 6460 .Case("avx512pf", X86Features::AVX512PF) 6461 .Case("avx512vbmi", X86Features::AVX512VBMI) 6462 .Case("avx512ifma", X86Features::AVX512IFMA) 6463 .Default(X86Features::MAX); 6464 assert(Feature != X86Features::MAX && "Invalid feature!"); 6465 6466 // Matching the struct layout from the compiler-rt/libgcc structure that is 6467 // filled in: 6468 // unsigned int __cpu_vendor; 6469 // unsigned int __cpu_type; 6470 // unsigned int __cpu_subtype; 6471 // unsigned int __cpu_features[1]; 6472 llvm::Type *STy = llvm::StructType::get( 6473 Int32Ty, Int32Ty, Int32Ty, llvm::ArrayType::get(Int32Ty, 1), nullptr); 6474 6475 // Grab the global __cpu_model. 6476 llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy, "__cpu_model"); 6477 6478 // Grab the first (0th) element from the field __cpu_features off of the 6479 // global in the struct STy. 6480 Value *Idxs[] = { 6481 ConstantInt::get(Int32Ty, 0), 6482 ConstantInt::get(Int32Ty, 3), 6483 ConstantInt::get(Int32Ty, 0) 6484 }; 6485 Value *CpuFeatures = Builder.CreateGEP(STy, CpuModel, Idxs); 6486 Value *Features = Builder.CreateAlignedLoad(CpuFeatures, 6487 CharUnits::fromQuantity(4)); 6488 6489 // Check the value of the bit corresponding to the feature requested. 6490 Value *Bitset = Builder.CreateAnd( 6491 Features, llvm::ConstantInt::get(Int32Ty, 1ULL << Feature)); 6492 return Builder.CreateICmpNE(Bitset, llvm::ConstantInt::get(Int32Ty, 0)); 6493 } 6494 case X86::BI_mm_prefetch: { 6495 Value *Address = Ops[0]; 6496 Value *RW = ConstantInt::get(Int32Ty, 0); 6497 Value *Locality = Ops[1]; 6498 Value *Data = ConstantInt::get(Int32Ty, 1); 6499 Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 6500 return Builder.CreateCall(F, {Address, RW, Locality, Data}); 6501 } 6502 case X86::BI__builtin_ia32_undef128: 6503 case X86::BI__builtin_ia32_undef256: 6504 case X86::BI__builtin_ia32_undef512: 6505 return UndefValue::get(ConvertType(E->getType())); 6506 case X86::BI__builtin_ia32_vec_init_v8qi: 6507 case X86::BI__builtin_ia32_vec_init_v4hi: 6508 case X86::BI__builtin_ia32_vec_init_v2si: 6509 return Builder.CreateBitCast(BuildVector(Ops), 6510 llvm::Type::getX86_MMXTy(getLLVMContext())); 6511 case X86::BI__builtin_ia32_vec_ext_v2si: 6512 return Builder.CreateExtractElement(Ops[0], 6513 llvm::ConstantInt::get(Ops[1]->getType(), 0)); 6514 case X86::BI__builtin_ia32_ldmxcsr: { 6515 Address Tmp = CreateMemTemp(E->getArg(0)->getType()); 6516 Builder.CreateStore(Ops[0], Tmp); 6517 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr), 6518 Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy)); 6519 } 6520 case X86::BI__builtin_ia32_stmxcsr: { 6521 Address Tmp = CreateMemTemp(E->getType()); 6522 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr), 6523 Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy)); 6524 return Builder.CreateLoad(Tmp, "stmxcsr"); 6525 } 6526 case X86::BI__builtin_ia32_xsave: 6527 case X86::BI__builtin_ia32_xsave64: 6528 case X86::BI__builtin_ia32_xrstor: 6529 case X86::BI__builtin_ia32_xrstor64: 6530 case X86::BI__builtin_ia32_xsaveopt: 6531 case X86::BI__builtin_ia32_xsaveopt64: 6532 case X86::BI__builtin_ia32_xrstors: 6533 case X86::BI__builtin_ia32_xrstors64: 6534 case X86::BI__builtin_ia32_xsavec: 6535 case X86::BI__builtin_ia32_xsavec64: 6536 case X86::BI__builtin_ia32_xsaves: 6537 case X86::BI__builtin_ia32_xsaves64: { 6538 Intrinsic::ID ID; 6539 #define INTRINSIC_X86_XSAVE_ID(NAME) \ 6540 case X86::BI__builtin_ia32_##NAME: \ 6541 ID = Intrinsic::x86_##NAME; \ 6542 break 6543 switch (BuiltinID) { 6544 default: llvm_unreachable("Unsupported intrinsic!"); 6545 INTRINSIC_X86_XSAVE_ID(xsave); 6546 INTRINSIC_X86_XSAVE_ID(xsave64); 6547 INTRINSIC_X86_XSAVE_ID(xrstor); 6548 INTRINSIC_X86_XSAVE_ID(xrstor64); 6549 INTRINSIC_X86_XSAVE_ID(xsaveopt); 6550 INTRINSIC_X86_XSAVE_ID(xsaveopt64); 6551 INTRINSIC_X86_XSAVE_ID(xrstors); 6552 INTRINSIC_X86_XSAVE_ID(xrstors64); 6553 INTRINSIC_X86_XSAVE_ID(xsavec); 6554 INTRINSIC_X86_XSAVE_ID(xsavec64); 6555 INTRINSIC_X86_XSAVE_ID(xsaves); 6556 INTRINSIC_X86_XSAVE_ID(xsaves64); 6557 } 6558 #undef INTRINSIC_X86_XSAVE_ID 6559 Value *Mhi = Builder.CreateTrunc( 6560 Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty); 6561 Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty); 6562 Ops[1] = Mhi; 6563 Ops.push_back(Mlo); 6564 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); 6565 } 6566 case X86::BI__builtin_ia32_storedqudi128_mask: 6567 case X86::BI__builtin_ia32_storedqusi128_mask: 6568 case X86::BI__builtin_ia32_storedquhi128_mask: 6569 case X86::BI__builtin_ia32_storedquqi128_mask: 6570 case X86::BI__builtin_ia32_storeupd128_mask: 6571 case X86::BI__builtin_ia32_storeups128_mask: 6572 case X86::BI__builtin_ia32_storedqudi256_mask: 6573 case X86::BI__builtin_ia32_storedqusi256_mask: 6574 case X86::BI__builtin_ia32_storedquhi256_mask: 6575 case X86::BI__builtin_ia32_storedquqi256_mask: 6576 case X86::BI__builtin_ia32_storeupd256_mask: 6577 case X86::BI__builtin_ia32_storeups256_mask: 6578 case X86::BI__builtin_ia32_storedqudi512_mask: 6579 case X86::BI__builtin_ia32_storedqusi512_mask: 6580 case X86::BI__builtin_ia32_storedquhi512_mask: 6581 case X86::BI__builtin_ia32_storedquqi512_mask: 6582 case X86::BI__builtin_ia32_storeupd512_mask: 6583 case X86::BI__builtin_ia32_storeups512_mask: 6584 return EmitX86MaskedStore(*this, Ops, 1); 6585 6586 case X86::BI__builtin_ia32_movdqa32store128_mask: 6587 case X86::BI__builtin_ia32_movdqa64store128_mask: 6588 case X86::BI__builtin_ia32_storeaps128_mask: 6589 case X86::BI__builtin_ia32_storeapd128_mask: 6590 case X86::BI__builtin_ia32_movdqa32store256_mask: 6591 case X86::BI__builtin_ia32_movdqa64store256_mask: 6592 case X86::BI__builtin_ia32_storeaps256_mask: 6593 case X86::BI__builtin_ia32_storeapd256_mask: 6594 case X86::BI__builtin_ia32_movdqa32store512_mask: 6595 case X86::BI__builtin_ia32_movdqa64store512_mask: 6596 case X86::BI__builtin_ia32_storeaps512_mask: 6597 case X86::BI__builtin_ia32_storeapd512_mask: { 6598 unsigned Align = 6599 getContext().getTypeAlignInChars(E->getArg(1)->getType()).getQuantity(); 6600 return EmitX86MaskedStore(*this, Ops, Align); 6601 } 6602 case X86::BI__builtin_ia32_loadups128_mask: 6603 case X86::BI__builtin_ia32_loadups256_mask: 6604 case X86::BI__builtin_ia32_loadups512_mask: 6605 case X86::BI__builtin_ia32_loadupd128_mask: 6606 case X86::BI__builtin_ia32_loadupd256_mask: 6607 case X86::BI__builtin_ia32_loadupd512_mask: 6608 case X86::BI__builtin_ia32_loaddquqi128_mask: 6609 case X86::BI__builtin_ia32_loaddquqi256_mask: 6610 case X86::BI__builtin_ia32_loaddquqi512_mask: 6611 case X86::BI__builtin_ia32_loaddquhi128_mask: 6612 case X86::BI__builtin_ia32_loaddquhi256_mask: 6613 case X86::BI__builtin_ia32_loaddquhi512_mask: 6614 case X86::BI__builtin_ia32_loaddqusi128_mask: 6615 case X86::BI__builtin_ia32_loaddqusi256_mask: 6616 case X86::BI__builtin_ia32_loaddqusi512_mask: 6617 case X86::BI__builtin_ia32_loaddqudi128_mask: 6618 case X86::BI__builtin_ia32_loaddqudi256_mask: 6619 case X86::BI__builtin_ia32_loaddqudi512_mask: 6620 return EmitX86MaskedLoad(*this, Ops, 1); 6621 6622 case X86::BI__builtin_ia32_loadaps128_mask: 6623 case X86::BI__builtin_ia32_loadaps256_mask: 6624 case X86::BI__builtin_ia32_loadaps512_mask: 6625 case X86::BI__builtin_ia32_loadapd128_mask: 6626 case X86::BI__builtin_ia32_loadapd256_mask: 6627 case X86::BI__builtin_ia32_loadapd512_mask: 6628 case X86::BI__builtin_ia32_movdqa32load128_mask: 6629 case X86::BI__builtin_ia32_movdqa32load256_mask: 6630 case X86::BI__builtin_ia32_movdqa32load512_mask: 6631 case X86::BI__builtin_ia32_movdqa64load128_mask: 6632 case X86::BI__builtin_ia32_movdqa64load256_mask: 6633 case X86::BI__builtin_ia32_movdqa64load512_mask: { 6634 unsigned Align = 6635 getContext().getTypeAlignInChars(E->getArg(1)->getType()).getQuantity(); 6636 return EmitX86MaskedLoad(*this, Ops, Align); 6637 } 6638 case X86::BI__builtin_ia32_storehps: 6639 case X86::BI__builtin_ia32_storelps: { 6640 llvm::Type *PtrTy = llvm::PointerType::getUnqual(Int64Ty); 6641 llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2); 6642 6643 // cast val v2i64 6644 Ops[1] = Builder.CreateBitCast(Ops[1], VecTy, "cast"); 6645 6646 // extract (0, 1) 6647 unsigned Index = BuiltinID == X86::BI__builtin_ia32_storelps ? 0 : 1; 6648 llvm::Value *Idx = llvm::ConstantInt::get(SizeTy, Index); 6649 Ops[1] = Builder.CreateExtractElement(Ops[1], Idx, "extract"); 6650 6651 // cast pointer to i64 & store 6652 Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy); 6653 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6654 } 6655 case X86::BI__builtin_ia32_palignr128: 6656 case X86::BI__builtin_ia32_palignr256: { 6657 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); 6658 6659 unsigned NumElts = 6660 cast<llvm::VectorType>(Ops[0]->getType())->getNumElements(); 6661 assert(NumElts % 16 == 0); 6662 6663 // If palignr is shifting the pair of vectors more than the size of two 6664 // lanes, emit zero. 6665 if (ShiftVal >= 32) 6666 return llvm::Constant::getNullValue(ConvertType(E->getType())); 6667 6668 // If palignr is shifting the pair of input vectors more than one lane, 6669 // but less than two lanes, convert to shifting in zeroes. 6670 if (ShiftVal > 16) { 6671 ShiftVal -= 16; 6672 Ops[1] = Ops[0]; 6673 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType()); 6674 } 6675 6676 int Indices[32]; 6677 // 256-bit palignr operates on 128-bit lanes so we need to handle that 6678 for (unsigned l = 0; l != NumElts; l += 16) { 6679 for (unsigned i = 0; i != 16; ++i) { 6680 unsigned Idx = ShiftVal + i; 6681 if (Idx >= 16) 6682 Idx += NumElts - 16; // End of lane, switch operand. 6683 Indices[l + i] = Idx + l; 6684 } 6685 } 6686 6687 return Builder.CreateShuffleVector(Ops[1], Ops[0], 6688 makeArrayRef(Indices, NumElts), 6689 "palignr"); 6690 } 6691 case X86::BI__builtin_ia32_pslldqi256: { 6692 // Shift value is in bits so divide by 8. 6693 unsigned shiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() >> 3; 6694 6695 // If pslldq is shifting the vector more than 15 bytes, emit zero. 6696 if (shiftVal >= 16) 6697 return llvm::Constant::getNullValue(ConvertType(E->getType())); 6698 6699 int Indices[32]; 6700 // 256-bit pslldq operates on 128-bit lanes so we need to handle that 6701 for (unsigned l = 0; l != 32; l += 16) { 6702 for (unsigned i = 0; i != 16; ++i) { 6703 unsigned Idx = 32 + i - shiftVal; 6704 if (Idx < 32) Idx -= 16; // end of lane, switch operand. 6705 Indices[l + i] = Idx + l; 6706 } 6707 } 6708 6709 llvm::Type *VecTy = llvm::VectorType::get(Int8Ty, 32); 6710 Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); 6711 Value *Zero = llvm::Constant::getNullValue(VecTy); 6712 6713 Value *SV = Builder.CreateShuffleVector(Zero, Ops[0], Indices, "pslldq"); 6714 llvm::Type *ResultType = ConvertType(E->getType()); 6715 return Builder.CreateBitCast(SV, ResultType, "cast"); 6716 } 6717 case X86::BI__builtin_ia32_psrldqi256: { 6718 // Shift value is in bits so divide by 8. 6719 unsigned shiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() >> 3; 6720 6721 // If psrldq is shifting the vector more than 15 bytes, emit zero. 6722 if (shiftVal >= 16) 6723 return llvm::Constant::getNullValue(ConvertType(E->getType())); 6724 6725 int Indices[32]; 6726 // 256-bit psrldq operates on 128-bit lanes so we need to handle that 6727 for (unsigned l = 0; l != 32; l += 16) { 6728 for (unsigned i = 0; i != 16; ++i) { 6729 unsigned Idx = i + shiftVal; 6730 if (Idx >= 16) Idx += 16; // end of lane, switch operand. 6731 Indices[l + i] = Idx + l; 6732 } 6733 } 6734 6735 llvm::Type *VecTy = llvm::VectorType::get(Int8Ty, 32); 6736 Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); 6737 Value *Zero = llvm::Constant::getNullValue(VecTy); 6738 6739 Value *SV = Builder.CreateShuffleVector(Ops[0], Zero, Indices, "psrldq"); 6740 llvm::Type *ResultType = ConvertType(E->getType()); 6741 return Builder.CreateBitCast(SV, ResultType, "cast"); 6742 } 6743 case X86::BI__builtin_ia32_movntps: 6744 case X86::BI__builtin_ia32_movntps256: 6745 case X86::BI__builtin_ia32_movntpd: 6746 case X86::BI__builtin_ia32_movntpd256: 6747 case X86::BI__builtin_ia32_movntdq: 6748 case X86::BI__builtin_ia32_movntdq256: 6749 case X86::BI__builtin_ia32_movnti: 6750 case X86::BI__builtin_ia32_movnti64: { 6751 llvm::MDNode *Node = llvm::MDNode::get( 6752 getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1))); 6753 6754 // Convert the type of the pointer to a pointer to the stored type. 6755 Value *BC = Builder.CreateBitCast(Ops[0], 6756 llvm::PointerType::getUnqual(Ops[1]->getType()), 6757 "cast"); 6758 StoreInst *SI = Builder.CreateDefaultAlignedStore(Ops[1], BC); 6759 SI->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node); 6760 6761 // If the operand is an integer, we can't assume alignment. Otherwise, 6762 // assume natural alignment. 6763 QualType ArgTy = E->getArg(1)->getType(); 6764 unsigned Align; 6765 if (ArgTy->isIntegerType()) 6766 Align = 1; 6767 else 6768 Align = getContext().getTypeSizeInChars(ArgTy).getQuantity(); 6769 SI->setAlignment(Align); 6770 return SI; 6771 } 6772 // 3DNow! 6773 case X86::BI__builtin_ia32_pswapdsf: 6774 case X86::BI__builtin_ia32_pswapdsi: { 6775 llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext()); 6776 Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast"); 6777 llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_3dnowa_pswapd); 6778 return Builder.CreateCall(F, Ops, "pswapd"); 6779 } 6780 case X86::BI__builtin_ia32_rdrand16_step: 6781 case X86::BI__builtin_ia32_rdrand32_step: 6782 case X86::BI__builtin_ia32_rdrand64_step: 6783 case X86::BI__builtin_ia32_rdseed16_step: 6784 case X86::BI__builtin_ia32_rdseed32_step: 6785 case X86::BI__builtin_ia32_rdseed64_step: { 6786 Intrinsic::ID ID; 6787 switch (BuiltinID) { 6788 default: llvm_unreachable("Unsupported intrinsic!"); 6789 case X86::BI__builtin_ia32_rdrand16_step: 6790 ID = Intrinsic::x86_rdrand_16; 6791 break; 6792 case X86::BI__builtin_ia32_rdrand32_step: 6793 ID = Intrinsic::x86_rdrand_32; 6794 break; 6795 case X86::BI__builtin_ia32_rdrand64_step: 6796 ID = Intrinsic::x86_rdrand_64; 6797 break; 6798 case X86::BI__builtin_ia32_rdseed16_step: 6799 ID = Intrinsic::x86_rdseed_16; 6800 break; 6801 case X86::BI__builtin_ia32_rdseed32_step: 6802 ID = Intrinsic::x86_rdseed_32; 6803 break; 6804 case X86::BI__builtin_ia32_rdseed64_step: 6805 ID = Intrinsic::x86_rdseed_64; 6806 break; 6807 } 6808 6809 Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID)); 6810 Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 0), 6811 Ops[0]); 6812 return Builder.CreateExtractValue(Call, 1); 6813 } 6814 // SSE comparison intrisics 6815 case X86::BI__builtin_ia32_cmpeqps: 6816 case X86::BI__builtin_ia32_cmpltps: 6817 case X86::BI__builtin_ia32_cmpleps: 6818 case X86::BI__builtin_ia32_cmpunordps: 6819 case X86::BI__builtin_ia32_cmpneqps: 6820 case X86::BI__builtin_ia32_cmpnltps: 6821 case X86::BI__builtin_ia32_cmpnleps: 6822 case X86::BI__builtin_ia32_cmpordps: 6823 case X86::BI__builtin_ia32_cmpeqss: 6824 case X86::BI__builtin_ia32_cmpltss: 6825 case X86::BI__builtin_ia32_cmpless: 6826 case X86::BI__builtin_ia32_cmpunordss: 6827 case X86::BI__builtin_ia32_cmpneqss: 6828 case X86::BI__builtin_ia32_cmpnltss: 6829 case X86::BI__builtin_ia32_cmpnless: 6830 case X86::BI__builtin_ia32_cmpordss: 6831 case X86::BI__builtin_ia32_cmpeqpd: 6832 case X86::BI__builtin_ia32_cmpltpd: 6833 case X86::BI__builtin_ia32_cmplepd: 6834 case X86::BI__builtin_ia32_cmpunordpd: 6835 case X86::BI__builtin_ia32_cmpneqpd: 6836 case X86::BI__builtin_ia32_cmpnltpd: 6837 case X86::BI__builtin_ia32_cmpnlepd: 6838 case X86::BI__builtin_ia32_cmpordpd: 6839 case X86::BI__builtin_ia32_cmpeqsd: 6840 case X86::BI__builtin_ia32_cmpltsd: 6841 case X86::BI__builtin_ia32_cmplesd: 6842 case X86::BI__builtin_ia32_cmpunordsd: 6843 case X86::BI__builtin_ia32_cmpneqsd: 6844 case X86::BI__builtin_ia32_cmpnltsd: 6845 case X86::BI__builtin_ia32_cmpnlesd: 6846 case X86::BI__builtin_ia32_cmpordsd: 6847 // These exist so that the builtin that takes an immediate can be bounds 6848 // checked by clang to avoid passing bad immediates to the backend. Since 6849 // AVX has a larger immediate than SSE we would need separate builtins to 6850 // do the different bounds checking. Rather than create a clang specific 6851 // SSE only builtin, this implements eight separate builtins to match gcc 6852 // implementation. 6853 6854 // Choose the immediate. 6855 unsigned Imm; 6856 switch (BuiltinID) { 6857 default: llvm_unreachable("Unsupported intrinsic!"); 6858 case X86::BI__builtin_ia32_cmpeqps: 6859 case X86::BI__builtin_ia32_cmpeqss: 6860 case X86::BI__builtin_ia32_cmpeqpd: 6861 case X86::BI__builtin_ia32_cmpeqsd: 6862 Imm = 0; 6863 break; 6864 case X86::BI__builtin_ia32_cmpltps: 6865 case X86::BI__builtin_ia32_cmpltss: 6866 case X86::BI__builtin_ia32_cmpltpd: 6867 case X86::BI__builtin_ia32_cmpltsd: 6868 Imm = 1; 6869 break; 6870 case X86::BI__builtin_ia32_cmpleps: 6871 case X86::BI__builtin_ia32_cmpless: 6872 case X86::BI__builtin_ia32_cmplepd: 6873 case X86::BI__builtin_ia32_cmplesd: 6874 Imm = 2; 6875 break; 6876 case X86::BI__builtin_ia32_cmpunordps: 6877 case X86::BI__builtin_ia32_cmpunordss: 6878 case X86::BI__builtin_ia32_cmpunordpd: 6879 case X86::BI__builtin_ia32_cmpunordsd: 6880 Imm = 3; 6881 break; 6882 case X86::BI__builtin_ia32_cmpneqps: 6883 case X86::BI__builtin_ia32_cmpneqss: 6884 case X86::BI__builtin_ia32_cmpneqpd: 6885 case X86::BI__builtin_ia32_cmpneqsd: 6886 Imm = 4; 6887 break; 6888 case X86::BI__builtin_ia32_cmpnltps: 6889 case X86::BI__builtin_ia32_cmpnltss: 6890 case X86::BI__builtin_ia32_cmpnltpd: 6891 case X86::BI__builtin_ia32_cmpnltsd: 6892 Imm = 5; 6893 break; 6894 case X86::BI__builtin_ia32_cmpnleps: 6895 case X86::BI__builtin_ia32_cmpnless: 6896 case X86::BI__builtin_ia32_cmpnlepd: 6897 case X86::BI__builtin_ia32_cmpnlesd: 6898 Imm = 6; 6899 break; 6900 case X86::BI__builtin_ia32_cmpordps: 6901 case X86::BI__builtin_ia32_cmpordss: 6902 case X86::BI__builtin_ia32_cmpordpd: 6903 case X86::BI__builtin_ia32_cmpordsd: 6904 Imm = 7; 6905 break; 6906 } 6907 6908 // Choose the intrinsic ID. 6909 const char *name; 6910 Intrinsic::ID ID; 6911 switch (BuiltinID) { 6912 default: llvm_unreachable("Unsupported intrinsic!"); 6913 case X86::BI__builtin_ia32_cmpeqps: 6914 case X86::BI__builtin_ia32_cmpltps: 6915 case X86::BI__builtin_ia32_cmpleps: 6916 case X86::BI__builtin_ia32_cmpunordps: 6917 case X86::BI__builtin_ia32_cmpneqps: 6918 case X86::BI__builtin_ia32_cmpnltps: 6919 case X86::BI__builtin_ia32_cmpnleps: 6920 case X86::BI__builtin_ia32_cmpordps: 6921 name = "cmpps"; 6922 ID = Intrinsic::x86_sse_cmp_ps; 6923 break; 6924 case X86::BI__builtin_ia32_cmpeqss: 6925 case X86::BI__builtin_ia32_cmpltss: 6926 case X86::BI__builtin_ia32_cmpless: 6927 case X86::BI__builtin_ia32_cmpunordss: 6928 case X86::BI__builtin_ia32_cmpneqss: 6929 case X86::BI__builtin_ia32_cmpnltss: 6930 case X86::BI__builtin_ia32_cmpnless: 6931 case X86::BI__builtin_ia32_cmpordss: 6932 name = "cmpss"; 6933 ID = Intrinsic::x86_sse_cmp_ss; 6934 break; 6935 case X86::BI__builtin_ia32_cmpeqpd: 6936 case X86::BI__builtin_ia32_cmpltpd: 6937 case X86::BI__builtin_ia32_cmplepd: 6938 case X86::BI__builtin_ia32_cmpunordpd: 6939 case X86::BI__builtin_ia32_cmpneqpd: 6940 case X86::BI__builtin_ia32_cmpnltpd: 6941 case X86::BI__builtin_ia32_cmpnlepd: 6942 case X86::BI__builtin_ia32_cmpordpd: 6943 name = "cmppd"; 6944 ID = Intrinsic::x86_sse2_cmp_pd; 6945 break; 6946 case X86::BI__builtin_ia32_cmpeqsd: 6947 case X86::BI__builtin_ia32_cmpltsd: 6948 case X86::BI__builtin_ia32_cmplesd: 6949 case X86::BI__builtin_ia32_cmpunordsd: 6950 case X86::BI__builtin_ia32_cmpneqsd: 6951 case X86::BI__builtin_ia32_cmpnltsd: 6952 case X86::BI__builtin_ia32_cmpnlesd: 6953 case X86::BI__builtin_ia32_cmpordsd: 6954 name = "cmpsd"; 6955 ID = Intrinsic::x86_sse2_cmp_sd; 6956 break; 6957 } 6958 6959 Ops.push_back(llvm::ConstantInt::get(Int8Ty, Imm)); 6960 llvm::Function *F = CGM.getIntrinsic(ID); 6961 return Builder.CreateCall(F, Ops, name); 6962 } 6963 } 6964 6965 6966 Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, 6967 const CallExpr *E) { 6968 SmallVector<Value*, 4> Ops; 6969 6970 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) 6971 Ops.push_back(EmitScalarExpr(E->getArg(i))); 6972 6973 Intrinsic::ID ID = Intrinsic::not_intrinsic; 6974 6975 switch (BuiltinID) { 6976 default: return nullptr; 6977 6978 // __builtin_ppc_get_timebase is GCC 4.8+'s PowerPC-specific name for what we 6979 // call __builtin_readcyclecounter. 6980 case PPC::BI__builtin_ppc_get_timebase: 6981 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::readcyclecounter)); 6982 6983 // vec_ld, vec_lvsl, vec_lvsr 6984 case PPC::BI__builtin_altivec_lvx: 6985 case PPC::BI__builtin_altivec_lvxl: 6986 case PPC::BI__builtin_altivec_lvebx: 6987 case PPC::BI__builtin_altivec_lvehx: 6988 case PPC::BI__builtin_altivec_lvewx: 6989 case PPC::BI__builtin_altivec_lvsl: 6990 case PPC::BI__builtin_altivec_lvsr: 6991 case PPC::BI__builtin_vsx_lxvd2x: 6992 case PPC::BI__builtin_vsx_lxvw4x: 6993 { 6994 Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); 6995 6996 Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]); 6997 Ops.pop_back(); 6998 6999 switch (BuiltinID) { 7000 default: llvm_unreachable("Unsupported ld/lvsl/lvsr intrinsic!"); 7001 case PPC::BI__builtin_altivec_lvx: 7002 ID = Intrinsic::ppc_altivec_lvx; 7003 break; 7004 case PPC::BI__builtin_altivec_lvxl: 7005 ID = Intrinsic::ppc_altivec_lvxl; 7006 break; 7007 case PPC::BI__builtin_altivec_lvebx: 7008 ID = Intrinsic::ppc_altivec_lvebx; 7009 break; 7010 case PPC::BI__builtin_altivec_lvehx: 7011 ID = Intrinsic::ppc_altivec_lvehx; 7012 break; 7013 case PPC::BI__builtin_altivec_lvewx: 7014 ID = Intrinsic::ppc_altivec_lvewx; 7015 break; 7016 case PPC::BI__builtin_altivec_lvsl: 7017 ID = Intrinsic::ppc_altivec_lvsl; 7018 break; 7019 case PPC::BI__builtin_altivec_lvsr: 7020 ID = Intrinsic::ppc_altivec_lvsr; 7021 break; 7022 case PPC::BI__builtin_vsx_lxvd2x: 7023 ID = Intrinsic::ppc_vsx_lxvd2x; 7024 break; 7025 case PPC::BI__builtin_vsx_lxvw4x: 7026 ID = Intrinsic::ppc_vsx_lxvw4x; 7027 break; 7028 } 7029 llvm::Function *F = CGM.getIntrinsic(ID); 7030 return Builder.CreateCall(F, Ops, ""); 7031 } 7032 7033 // vec_st 7034 case PPC::BI__builtin_altivec_stvx: 7035 case PPC::BI__builtin_altivec_stvxl: 7036 case PPC::BI__builtin_altivec_stvebx: 7037 case PPC::BI__builtin_altivec_stvehx: 7038 case PPC::BI__builtin_altivec_stvewx: 7039 case PPC::BI__builtin_vsx_stxvd2x: 7040 case PPC::BI__builtin_vsx_stxvw4x: 7041 { 7042 Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy); 7043 Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]); 7044 Ops.pop_back(); 7045 7046 switch (BuiltinID) { 7047 default: llvm_unreachable("Unsupported st intrinsic!"); 7048 case PPC::BI__builtin_altivec_stvx: 7049 ID = Intrinsic::ppc_altivec_stvx; 7050 break; 7051 case PPC::BI__builtin_altivec_stvxl: 7052 ID = Intrinsic::ppc_altivec_stvxl; 7053 break; 7054 case PPC::BI__builtin_altivec_stvebx: 7055 ID = Intrinsic::ppc_altivec_stvebx; 7056 break; 7057 case PPC::BI__builtin_altivec_stvehx: 7058 ID = Intrinsic::ppc_altivec_stvehx; 7059 break; 7060 case PPC::BI__builtin_altivec_stvewx: 7061 ID = Intrinsic::ppc_altivec_stvewx; 7062 break; 7063 case PPC::BI__builtin_vsx_stxvd2x: 7064 ID = Intrinsic::ppc_vsx_stxvd2x; 7065 break; 7066 case PPC::BI__builtin_vsx_stxvw4x: 7067 ID = Intrinsic::ppc_vsx_stxvw4x; 7068 break; 7069 } 7070 llvm::Function *F = CGM.getIntrinsic(ID); 7071 return Builder.CreateCall(F, Ops, ""); 7072 } 7073 // Square root 7074 case PPC::BI__builtin_vsx_xvsqrtsp: 7075 case PPC::BI__builtin_vsx_xvsqrtdp: { 7076 llvm::Type *ResultType = ConvertType(E->getType()); 7077 Value *X = EmitScalarExpr(E->getArg(0)); 7078 ID = Intrinsic::sqrt; 7079 llvm::Function *F = CGM.getIntrinsic(ID, ResultType); 7080 return Builder.CreateCall(F, X); 7081 } 7082 // Count leading zeros 7083 case PPC::BI__builtin_altivec_vclzb: 7084 case PPC::BI__builtin_altivec_vclzh: 7085 case PPC::BI__builtin_altivec_vclzw: 7086 case PPC::BI__builtin_altivec_vclzd: { 7087 llvm::Type *ResultType = ConvertType(E->getType()); 7088 Value *X = EmitScalarExpr(E->getArg(0)); 7089 Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false); 7090 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType); 7091 return Builder.CreateCall(F, {X, Undef}); 7092 } 7093 // Copy sign 7094 case PPC::BI__builtin_vsx_xvcpsgnsp: 7095 case PPC::BI__builtin_vsx_xvcpsgndp: { 7096 llvm::Type *ResultType = ConvertType(E->getType()); 7097 Value *X = EmitScalarExpr(E->getArg(0)); 7098 Value *Y = EmitScalarExpr(E->getArg(1)); 7099 ID = Intrinsic::copysign; 7100 llvm::Function *F = CGM.getIntrinsic(ID, ResultType); 7101 return Builder.CreateCall(F, {X, Y}); 7102 } 7103 // Rounding/truncation 7104 case PPC::BI__builtin_vsx_xvrspip: 7105 case PPC::BI__builtin_vsx_xvrdpip: 7106 case PPC::BI__builtin_vsx_xvrdpim: 7107 case PPC::BI__builtin_vsx_xvrspim: 7108 case PPC::BI__builtin_vsx_xvrdpi: 7109 case PPC::BI__builtin_vsx_xvrspi: 7110 case PPC::BI__builtin_vsx_xvrdpic: 7111 case PPC::BI__builtin_vsx_xvrspic: 7112 case PPC::BI__builtin_vsx_xvrdpiz: 7113 case PPC::BI__builtin_vsx_xvrspiz: { 7114 llvm::Type *ResultType = ConvertType(E->getType()); 7115 Value *X = EmitScalarExpr(E->getArg(0)); 7116 if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim || 7117 BuiltinID == PPC::BI__builtin_vsx_xvrspim) 7118 ID = Intrinsic::floor; 7119 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi || 7120 BuiltinID == PPC::BI__builtin_vsx_xvrspi) 7121 ID = Intrinsic::round; 7122 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic || 7123 BuiltinID == PPC::BI__builtin_vsx_xvrspic) 7124 ID = Intrinsic::nearbyint; 7125 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip || 7126 BuiltinID == PPC::BI__builtin_vsx_xvrspip) 7127 ID = Intrinsic::ceil; 7128 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz || 7129 BuiltinID == PPC::BI__builtin_vsx_xvrspiz) 7130 ID = Intrinsic::trunc; 7131 llvm::Function *F = CGM.getIntrinsic(ID, ResultType); 7132 return Builder.CreateCall(F, X); 7133 } 7134 7135 // Absolute value 7136 case PPC::BI__builtin_vsx_xvabsdp: 7137 case PPC::BI__builtin_vsx_xvabssp: { 7138 llvm::Type *ResultType = ConvertType(E->getType()); 7139 Value *X = EmitScalarExpr(E->getArg(0)); 7140 llvm::Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType); 7141 return Builder.CreateCall(F, X); 7142 } 7143 7144 // FMA variations 7145 case PPC::BI__builtin_vsx_xvmaddadp: 7146 case PPC::BI__builtin_vsx_xvmaddasp: 7147 case PPC::BI__builtin_vsx_xvnmaddadp: 7148 case PPC::BI__builtin_vsx_xvnmaddasp: 7149 case PPC::BI__builtin_vsx_xvmsubadp: 7150 case PPC::BI__builtin_vsx_xvmsubasp: 7151 case PPC::BI__builtin_vsx_xvnmsubadp: 7152 case PPC::BI__builtin_vsx_xvnmsubasp: { 7153 llvm::Type *ResultType = ConvertType(E->getType()); 7154 Value *X = EmitScalarExpr(E->getArg(0)); 7155 Value *Y = EmitScalarExpr(E->getArg(1)); 7156 Value *Z = EmitScalarExpr(E->getArg(2)); 7157 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType); 7158 llvm::Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType); 7159 switch (BuiltinID) { 7160 case PPC::BI__builtin_vsx_xvmaddadp: 7161 case PPC::BI__builtin_vsx_xvmaddasp: 7162 return Builder.CreateCall(F, {X, Y, Z}); 7163 case PPC::BI__builtin_vsx_xvnmaddadp: 7164 case PPC::BI__builtin_vsx_xvnmaddasp: 7165 return Builder.CreateFSub(Zero, 7166 Builder.CreateCall(F, {X, Y, Z}), "sub"); 7167 case PPC::BI__builtin_vsx_xvmsubadp: 7168 case PPC::BI__builtin_vsx_xvmsubasp: 7169 return Builder.CreateCall(F, 7170 {X, Y, Builder.CreateFSub(Zero, Z, "sub")}); 7171 case PPC::BI__builtin_vsx_xvnmsubadp: 7172 case PPC::BI__builtin_vsx_xvnmsubasp: 7173 Value *FsubRes = 7174 Builder.CreateCall(F, {X, Y, Builder.CreateFSub(Zero, Z, "sub")}); 7175 return Builder.CreateFSub(Zero, FsubRes, "sub"); 7176 } 7177 llvm_unreachable("Unknown FMA operation"); 7178 return nullptr; // Suppress no-return warning 7179 } 7180 } 7181 } 7182 7183 Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, 7184 const CallExpr *E) { 7185 switch (BuiltinID) { 7186 case AMDGPU::BI__builtin_amdgcn_div_scale: 7187 case AMDGPU::BI__builtin_amdgcn_div_scalef: { 7188 // Translate from the intrinsics's struct return to the builtin's out 7189 // argument. 7190 7191 Address FlagOutPtr = EmitPointerWithAlignment(E->getArg(3)); 7192 7193 llvm::Value *X = EmitScalarExpr(E->getArg(0)); 7194 llvm::Value *Y = EmitScalarExpr(E->getArg(1)); 7195 llvm::Value *Z = EmitScalarExpr(E->getArg(2)); 7196 7197 llvm::Value *Callee = CGM.getIntrinsic(Intrinsic::amdgcn_div_scale, 7198 X->getType()); 7199 7200 llvm::Value *Tmp = Builder.CreateCall(Callee, {X, Y, Z}); 7201 7202 llvm::Value *Result = Builder.CreateExtractValue(Tmp, 0); 7203 llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1); 7204 7205 llvm::Type *RealFlagType 7206 = FlagOutPtr.getPointer()->getType()->getPointerElementType(); 7207 7208 llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType); 7209 Builder.CreateStore(FlagExt, FlagOutPtr); 7210 return Result; 7211 } 7212 case AMDGPU::BI__builtin_amdgcn_div_fmas: 7213 case AMDGPU::BI__builtin_amdgcn_div_fmasf: { 7214 llvm::Value *Src0 = EmitScalarExpr(E->getArg(0)); 7215 llvm::Value *Src1 = EmitScalarExpr(E->getArg(1)); 7216 llvm::Value *Src2 = EmitScalarExpr(E->getArg(2)); 7217 llvm::Value *Src3 = EmitScalarExpr(E->getArg(3)); 7218 7219 llvm::Value *F = CGM.getIntrinsic(Intrinsic::amdgcn_div_fmas, 7220 Src0->getType()); 7221 llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3); 7222 return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool}); 7223 } 7224 case AMDGPU::BI__builtin_amdgcn_div_fixup: 7225 case AMDGPU::BI__builtin_amdgcn_div_fixupf: 7226 return emitTernaryFPBuiltin(*this, E, Intrinsic::amdgcn_div_fixup); 7227 case AMDGPU::BI__builtin_amdgcn_trig_preop: 7228 case AMDGPU::BI__builtin_amdgcn_trig_preopf: 7229 return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_trig_preop); 7230 case AMDGPU::BI__builtin_amdgcn_rcp: 7231 case AMDGPU::BI__builtin_amdgcn_rcpf: 7232 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rcp); 7233 case AMDGPU::BI__builtin_amdgcn_rsq: 7234 case AMDGPU::BI__builtin_amdgcn_rsqf: 7235 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq); 7236 case AMDGPU::BI__builtin_amdgcn_rsq_clamp: 7237 case AMDGPU::BI__builtin_amdgcn_rsq_clampf: 7238 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq_clamp); 7239 case AMDGPU::BI__builtin_amdgcn_sinf: 7240 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_sin); 7241 case AMDGPU::BI__builtin_amdgcn_cosf: 7242 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_cos); 7243 case AMDGPU::BI__builtin_amdgcn_log_clampf: 7244 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_log_clamp); 7245 case AMDGPU::BI__builtin_amdgcn_ldexp: 7246 case AMDGPU::BI__builtin_amdgcn_ldexpf: 7247 return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_ldexp); 7248 case AMDGPU::BI__builtin_amdgcn_frexp_mant: 7249 case AMDGPU::BI__builtin_amdgcn_frexp_mantf: { 7250 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_frexp_mant); 7251 } 7252 case AMDGPU::BI__builtin_amdgcn_frexp_exp: 7253 case AMDGPU::BI__builtin_amdgcn_frexp_expf: { 7254 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_frexp_exp); 7255 } 7256 case AMDGPU::BI__builtin_amdgcn_fract: 7257 case AMDGPU::BI__builtin_amdgcn_fractf: 7258 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_fract); 7259 case AMDGPU::BI__builtin_amdgcn_class: 7260 case AMDGPU::BI__builtin_amdgcn_classf: 7261 return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_class); 7262 7263 // Legacy amdgpu prefix 7264 case AMDGPU::BI__builtin_amdgpu_rsq: 7265 case AMDGPU::BI__builtin_amdgpu_rsqf: { 7266 if (getTarget().getTriple().getArch() == Triple::amdgcn) 7267 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq); 7268 return emitUnaryBuiltin(*this, E, Intrinsic::r600_rsq); 7269 } 7270 case AMDGPU::BI__builtin_amdgpu_ldexp: 7271 case AMDGPU::BI__builtin_amdgpu_ldexpf: { 7272 if (getTarget().getTriple().getArch() == Triple::amdgcn) 7273 return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_ldexp); 7274 return emitFPIntBuiltin(*this, E, Intrinsic::AMDGPU_ldexp); 7275 } 7276 default: 7277 return nullptr; 7278 } 7279 } 7280 7281 /// Handle a SystemZ function in which the final argument is a pointer 7282 /// to an int that receives the post-instruction CC value. At the LLVM level 7283 /// this is represented as a function that returns a {result, cc} pair. 7284 static Value *EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, 7285 unsigned IntrinsicID, 7286 const CallExpr *E) { 7287 unsigned NumArgs = E->getNumArgs() - 1; 7288 SmallVector<Value *, 8> Args(NumArgs); 7289 for (unsigned I = 0; I < NumArgs; ++I) 7290 Args[I] = CGF.EmitScalarExpr(E->getArg(I)); 7291 Address CCPtr = CGF.EmitPointerWithAlignment(E->getArg(NumArgs)); 7292 Value *F = CGF.CGM.getIntrinsic(IntrinsicID); 7293 Value *Call = CGF.Builder.CreateCall(F, Args); 7294 Value *CC = CGF.Builder.CreateExtractValue(Call, 1); 7295 CGF.Builder.CreateStore(CC, CCPtr); 7296 return CGF.Builder.CreateExtractValue(Call, 0); 7297 } 7298 7299 Value *CodeGenFunction::EmitSystemZBuiltinExpr(unsigned BuiltinID, 7300 const CallExpr *E) { 7301 switch (BuiltinID) { 7302 case SystemZ::BI__builtin_tbegin: { 7303 Value *TDB = EmitScalarExpr(E->getArg(0)); 7304 Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c); 7305 Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin); 7306 return Builder.CreateCall(F, {TDB, Control}); 7307 } 7308 case SystemZ::BI__builtin_tbegin_nofloat: { 7309 Value *TDB = EmitScalarExpr(E->getArg(0)); 7310 Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c); 7311 Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin_nofloat); 7312 return Builder.CreateCall(F, {TDB, Control}); 7313 } 7314 case SystemZ::BI__builtin_tbeginc: { 7315 Value *TDB = llvm::ConstantPointerNull::get(Int8PtrTy); 7316 Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff08); 7317 Value *F = CGM.getIntrinsic(Intrinsic::s390_tbeginc); 7318 return Builder.CreateCall(F, {TDB, Control}); 7319 } 7320 case SystemZ::BI__builtin_tabort: { 7321 Value *Data = EmitScalarExpr(E->getArg(0)); 7322 Value *F = CGM.getIntrinsic(Intrinsic::s390_tabort); 7323 return Builder.CreateCall(F, Builder.CreateSExt(Data, Int64Ty, "tabort")); 7324 } 7325 case SystemZ::BI__builtin_non_tx_store: { 7326 Value *Address = EmitScalarExpr(E->getArg(0)); 7327 Value *Data = EmitScalarExpr(E->getArg(1)); 7328 Value *F = CGM.getIntrinsic(Intrinsic::s390_ntstg); 7329 return Builder.CreateCall(F, {Data, Address}); 7330 } 7331 7332 // Vector builtins. Note that most vector builtins are mapped automatically 7333 // to target-specific LLVM intrinsics. The ones handled specially here can 7334 // be represented via standard LLVM IR, which is preferable to enable common 7335 // LLVM optimizations. 7336 7337 case SystemZ::BI__builtin_s390_vpopctb: 7338 case SystemZ::BI__builtin_s390_vpopcth: 7339 case SystemZ::BI__builtin_s390_vpopctf: 7340 case SystemZ::BI__builtin_s390_vpopctg: { 7341 llvm::Type *ResultType = ConvertType(E->getType()); 7342 Value *X = EmitScalarExpr(E->getArg(0)); 7343 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType); 7344 return Builder.CreateCall(F, X); 7345 } 7346 7347 case SystemZ::BI__builtin_s390_vclzb: 7348 case SystemZ::BI__builtin_s390_vclzh: 7349 case SystemZ::BI__builtin_s390_vclzf: 7350 case SystemZ::BI__builtin_s390_vclzg: { 7351 llvm::Type *ResultType = ConvertType(E->getType()); 7352 Value *X = EmitScalarExpr(E->getArg(0)); 7353 Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false); 7354 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType); 7355 return Builder.CreateCall(F, {X, Undef}); 7356 } 7357 7358 case SystemZ::BI__builtin_s390_vctzb: 7359 case SystemZ::BI__builtin_s390_vctzh: 7360 case SystemZ::BI__builtin_s390_vctzf: 7361 case SystemZ::BI__builtin_s390_vctzg: { 7362 llvm::Type *ResultType = ConvertType(E->getType()); 7363 Value *X = EmitScalarExpr(E->getArg(0)); 7364 Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false); 7365 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType); 7366 return Builder.CreateCall(F, {X, Undef}); 7367 } 7368 7369 case SystemZ::BI__builtin_s390_vfsqdb: { 7370 llvm::Type *ResultType = ConvertType(E->getType()); 7371 Value *X = EmitScalarExpr(E->getArg(0)); 7372 Function *F = CGM.getIntrinsic(Intrinsic::sqrt, ResultType); 7373 return Builder.CreateCall(F, X); 7374 } 7375 case SystemZ::BI__builtin_s390_vfmadb: { 7376 llvm::Type *ResultType = ConvertType(E->getType()); 7377 Value *X = EmitScalarExpr(E->getArg(0)); 7378 Value *Y = EmitScalarExpr(E->getArg(1)); 7379 Value *Z = EmitScalarExpr(E->getArg(2)); 7380 Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType); 7381 return Builder.CreateCall(F, {X, Y, Z}); 7382 } 7383 case SystemZ::BI__builtin_s390_vfmsdb: { 7384 llvm::Type *ResultType = ConvertType(E->getType()); 7385 Value *X = EmitScalarExpr(E->getArg(0)); 7386 Value *Y = EmitScalarExpr(E->getArg(1)); 7387 Value *Z = EmitScalarExpr(E->getArg(2)); 7388 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType); 7389 Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType); 7390 return Builder.CreateCall(F, {X, Y, Builder.CreateFSub(Zero, Z, "sub")}); 7391 } 7392 case SystemZ::BI__builtin_s390_vflpdb: { 7393 llvm::Type *ResultType = ConvertType(E->getType()); 7394 Value *X = EmitScalarExpr(E->getArg(0)); 7395 Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType); 7396 return Builder.CreateCall(F, X); 7397 } 7398 case SystemZ::BI__builtin_s390_vflndb: { 7399 llvm::Type *ResultType = ConvertType(E->getType()); 7400 Value *X = EmitScalarExpr(E->getArg(0)); 7401 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType); 7402 Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType); 7403 return Builder.CreateFSub(Zero, Builder.CreateCall(F, X), "sub"); 7404 } 7405 case SystemZ::BI__builtin_s390_vfidb: { 7406 llvm::Type *ResultType = ConvertType(E->getType()); 7407 Value *X = EmitScalarExpr(E->getArg(0)); 7408 // Constant-fold the M4 and M5 mask arguments. 7409 llvm::APSInt M4, M5; 7410 bool IsConstM4 = E->getArg(1)->isIntegerConstantExpr(M4, getContext()); 7411 bool IsConstM5 = E->getArg(2)->isIntegerConstantExpr(M5, getContext()); 7412 assert(IsConstM4 && IsConstM5 && "Constant arg isn't actually constant?"); 7413 (void)IsConstM4; (void)IsConstM5; 7414 // Check whether this instance of vfidb can be represented via a LLVM 7415 // standard intrinsic. We only support some combinations of M4 and M5. 7416 Intrinsic::ID ID = Intrinsic::not_intrinsic; 7417 switch (M4.getZExtValue()) { 7418 default: break; 7419 case 0: // IEEE-inexact exception allowed 7420 switch (M5.getZExtValue()) { 7421 default: break; 7422 case 0: ID = Intrinsic::rint; break; 7423 } 7424 break; 7425 case 4: // IEEE-inexact exception suppressed 7426 switch (M5.getZExtValue()) { 7427 default: break; 7428 case 0: ID = Intrinsic::nearbyint; break; 7429 case 1: ID = Intrinsic::round; break; 7430 case 5: ID = Intrinsic::trunc; break; 7431 case 6: ID = Intrinsic::ceil; break; 7432 case 7: ID = Intrinsic::floor; break; 7433 } 7434 break; 7435 } 7436 if (ID != Intrinsic::not_intrinsic) { 7437 Function *F = CGM.getIntrinsic(ID, ResultType); 7438 return Builder.CreateCall(F, X); 7439 } 7440 Function *F = CGM.getIntrinsic(Intrinsic::s390_vfidb); 7441 Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4); 7442 Value *M5Value = llvm::ConstantInt::get(getLLVMContext(), M5); 7443 return Builder.CreateCall(F, {X, M4Value, M5Value}); 7444 } 7445 7446 // Vector intrisincs that output the post-instruction CC value. 7447 7448 #define INTRINSIC_WITH_CC(NAME) \ 7449 case SystemZ::BI__builtin_##NAME: \ 7450 return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E) 7451 7452 INTRINSIC_WITH_CC(s390_vpkshs); 7453 INTRINSIC_WITH_CC(s390_vpksfs); 7454 INTRINSIC_WITH_CC(s390_vpksgs); 7455 7456 INTRINSIC_WITH_CC(s390_vpklshs); 7457 INTRINSIC_WITH_CC(s390_vpklsfs); 7458 INTRINSIC_WITH_CC(s390_vpklsgs); 7459 7460 INTRINSIC_WITH_CC(s390_vceqbs); 7461 INTRINSIC_WITH_CC(s390_vceqhs); 7462 INTRINSIC_WITH_CC(s390_vceqfs); 7463 INTRINSIC_WITH_CC(s390_vceqgs); 7464 7465 INTRINSIC_WITH_CC(s390_vchbs); 7466 INTRINSIC_WITH_CC(s390_vchhs); 7467 INTRINSIC_WITH_CC(s390_vchfs); 7468 INTRINSIC_WITH_CC(s390_vchgs); 7469 7470 INTRINSIC_WITH_CC(s390_vchlbs); 7471 INTRINSIC_WITH_CC(s390_vchlhs); 7472 INTRINSIC_WITH_CC(s390_vchlfs); 7473 INTRINSIC_WITH_CC(s390_vchlgs); 7474 7475 INTRINSIC_WITH_CC(s390_vfaebs); 7476 INTRINSIC_WITH_CC(s390_vfaehs); 7477 INTRINSIC_WITH_CC(s390_vfaefs); 7478 7479 INTRINSIC_WITH_CC(s390_vfaezbs); 7480 INTRINSIC_WITH_CC(s390_vfaezhs); 7481 INTRINSIC_WITH_CC(s390_vfaezfs); 7482 7483 INTRINSIC_WITH_CC(s390_vfeebs); 7484 INTRINSIC_WITH_CC(s390_vfeehs); 7485 INTRINSIC_WITH_CC(s390_vfeefs); 7486 7487 INTRINSIC_WITH_CC(s390_vfeezbs); 7488 INTRINSIC_WITH_CC(s390_vfeezhs); 7489 INTRINSIC_WITH_CC(s390_vfeezfs); 7490 7491 INTRINSIC_WITH_CC(s390_vfenebs); 7492 INTRINSIC_WITH_CC(s390_vfenehs); 7493 INTRINSIC_WITH_CC(s390_vfenefs); 7494 7495 INTRINSIC_WITH_CC(s390_vfenezbs); 7496 INTRINSIC_WITH_CC(s390_vfenezhs); 7497 INTRINSIC_WITH_CC(s390_vfenezfs); 7498 7499 INTRINSIC_WITH_CC(s390_vistrbs); 7500 INTRINSIC_WITH_CC(s390_vistrhs); 7501 INTRINSIC_WITH_CC(s390_vistrfs); 7502 7503 INTRINSIC_WITH_CC(s390_vstrcbs); 7504 INTRINSIC_WITH_CC(s390_vstrchs); 7505 INTRINSIC_WITH_CC(s390_vstrcfs); 7506 7507 INTRINSIC_WITH_CC(s390_vstrczbs); 7508 INTRINSIC_WITH_CC(s390_vstrczhs); 7509 INTRINSIC_WITH_CC(s390_vstrczfs); 7510 7511 INTRINSIC_WITH_CC(s390_vfcedbs); 7512 INTRINSIC_WITH_CC(s390_vfchdbs); 7513 INTRINSIC_WITH_CC(s390_vfchedbs); 7514 7515 INTRINSIC_WITH_CC(s390_vftcidb); 7516 7517 #undef INTRINSIC_WITH_CC 7518 7519 default: 7520 return nullptr; 7521 } 7522 } 7523 7524 Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, 7525 const CallExpr *E) { 7526 auto MakeLdg = [&](unsigned IntrinsicID) { 7527 Value *Ptr = EmitScalarExpr(E->getArg(0)); 7528 AlignmentSource AlignSource; 7529 clang::CharUnits Align = 7530 getNaturalPointeeTypeAlignment(E->getArg(0)->getType(), &AlignSource); 7531 return Builder.CreateCall( 7532 CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(), 7533 Ptr->getType()}), 7534 {Ptr, ConstantInt::get(Builder.getInt32Ty(), Align.getQuantity())}); 7535 }; 7536 7537 switch (BuiltinID) { 7538 case NVPTX::BI__nvvm_atom_add_gen_i: 7539 case NVPTX::BI__nvvm_atom_add_gen_l: 7540 case NVPTX::BI__nvvm_atom_add_gen_ll: 7541 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Add, E); 7542 7543 case NVPTX::BI__nvvm_atom_sub_gen_i: 7544 case NVPTX::BI__nvvm_atom_sub_gen_l: 7545 case NVPTX::BI__nvvm_atom_sub_gen_ll: 7546 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Sub, E); 7547 7548 case NVPTX::BI__nvvm_atom_and_gen_i: 7549 case NVPTX::BI__nvvm_atom_and_gen_l: 7550 case NVPTX::BI__nvvm_atom_and_gen_ll: 7551 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::And, E); 7552 7553 case NVPTX::BI__nvvm_atom_or_gen_i: 7554 case NVPTX::BI__nvvm_atom_or_gen_l: 7555 case NVPTX::BI__nvvm_atom_or_gen_ll: 7556 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Or, E); 7557 7558 case NVPTX::BI__nvvm_atom_xor_gen_i: 7559 case NVPTX::BI__nvvm_atom_xor_gen_l: 7560 case NVPTX::BI__nvvm_atom_xor_gen_ll: 7561 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xor, E); 7562 7563 case NVPTX::BI__nvvm_atom_xchg_gen_i: 7564 case NVPTX::BI__nvvm_atom_xchg_gen_l: 7565 case NVPTX::BI__nvvm_atom_xchg_gen_ll: 7566 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xchg, E); 7567 7568 case NVPTX::BI__nvvm_atom_max_gen_i: 7569 case NVPTX::BI__nvvm_atom_max_gen_l: 7570 case NVPTX::BI__nvvm_atom_max_gen_ll: 7571 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Max, E); 7572 7573 case NVPTX::BI__nvvm_atom_max_gen_ui: 7574 case NVPTX::BI__nvvm_atom_max_gen_ul: 7575 case NVPTX::BI__nvvm_atom_max_gen_ull: 7576 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMax, E); 7577 7578 case NVPTX::BI__nvvm_atom_min_gen_i: 7579 case NVPTX::BI__nvvm_atom_min_gen_l: 7580 case NVPTX::BI__nvvm_atom_min_gen_ll: 7581 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Min, E); 7582 7583 case NVPTX::BI__nvvm_atom_min_gen_ui: 7584 case NVPTX::BI__nvvm_atom_min_gen_ul: 7585 case NVPTX::BI__nvvm_atom_min_gen_ull: 7586 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMin, E); 7587 7588 case NVPTX::BI__nvvm_atom_cas_gen_i: 7589 case NVPTX::BI__nvvm_atom_cas_gen_l: 7590 case NVPTX::BI__nvvm_atom_cas_gen_ll: 7591 // __nvvm_atom_cas_gen_* should return the old value rather than the 7592 // success flag. 7593 return MakeAtomicCmpXchgValue(*this, E, /*ReturnBool=*/false); 7594 7595 case NVPTX::BI__nvvm_atom_add_gen_f: { 7596 Value *Ptr = EmitScalarExpr(E->getArg(0)); 7597 Value *Val = EmitScalarExpr(E->getArg(1)); 7598 // atomicrmw only deals with integer arguments so we need to use 7599 // LLVM's nvvm_atomic_load_add_f32 intrinsic for that. 7600 Value *FnALAF32 = 7601 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_add_f32, Ptr->getType()); 7602 return Builder.CreateCall(FnALAF32, {Ptr, Val}); 7603 } 7604 7605 case NVPTX::BI__nvvm_atom_inc_gen_ui: { 7606 Value *Ptr = EmitScalarExpr(E->getArg(0)); 7607 Value *Val = EmitScalarExpr(E->getArg(1)); 7608 Value *FnALI32 = 7609 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_inc_32, Ptr->getType()); 7610 return Builder.CreateCall(FnALI32, {Ptr, Val}); 7611 } 7612 7613 case NVPTX::BI__nvvm_atom_dec_gen_ui: { 7614 Value *Ptr = EmitScalarExpr(E->getArg(0)); 7615 Value *Val = EmitScalarExpr(E->getArg(1)); 7616 Value *FnALD32 = 7617 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_dec_32, Ptr->getType()); 7618 return Builder.CreateCall(FnALD32, {Ptr, Val}); 7619 } 7620 7621 case NVPTX::BI__nvvm_ldg_c: 7622 case NVPTX::BI__nvvm_ldg_c2: 7623 case NVPTX::BI__nvvm_ldg_c4: 7624 case NVPTX::BI__nvvm_ldg_s: 7625 case NVPTX::BI__nvvm_ldg_s2: 7626 case NVPTX::BI__nvvm_ldg_s4: 7627 case NVPTX::BI__nvvm_ldg_i: 7628 case NVPTX::BI__nvvm_ldg_i2: 7629 case NVPTX::BI__nvvm_ldg_i4: 7630 case NVPTX::BI__nvvm_ldg_l: 7631 case NVPTX::BI__nvvm_ldg_ll: 7632 case NVPTX::BI__nvvm_ldg_ll2: 7633 case NVPTX::BI__nvvm_ldg_uc: 7634 case NVPTX::BI__nvvm_ldg_uc2: 7635 case NVPTX::BI__nvvm_ldg_uc4: 7636 case NVPTX::BI__nvvm_ldg_us: 7637 case NVPTX::BI__nvvm_ldg_us2: 7638 case NVPTX::BI__nvvm_ldg_us4: 7639 case NVPTX::BI__nvvm_ldg_ui: 7640 case NVPTX::BI__nvvm_ldg_ui2: 7641 case NVPTX::BI__nvvm_ldg_ui4: 7642 case NVPTX::BI__nvvm_ldg_ul: 7643 case NVPTX::BI__nvvm_ldg_ull: 7644 case NVPTX::BI__nvvm_ldg_ull2: 7645 // PTX Interoperability section 2.2: "For a vector with an even number of 7646 // elements, its alignment is set to number of elements times the alignment 7647 // of its member: n*alignof(t)." 7648 return MakeLdg(Intrinsic::nvvm_ldg_global_i); 7649 case NVPTX::BI__nvvm_ldg_f: 7650 case NVPTX::BI__nvvm_ldg_f2: 7651 case NVPTX::BI__nvvm_ldg_f4: 7652 case NVPTX::BI__nvvm_ldg_d: 7653 case NVPTX::BI__nvvm_ldg_d2: 7654 return MakeLdg(Intrinsic::nvvm_ldg_global_f); 7655 default: 7656 return nullptr; 7657 } 7658 } 7659 7660 Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, 7661 const CallExpr *E) { 7662 switch (BuiltinID) { 7663 case WebAssembly::BI__builtin_wasm_current_memory: { 7664 llvm::Type *ResultType = ConvertType(E->getType()); 7665 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_current_memory, ResultType); 7666 return Builder.CreateCall(Callee); 7667 } 7668 case WebAssembly::BI__builtin_wasm_grow_memory: { 7669 Value *X = EmitScalarExpr(E->getArg(0)); 7670 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_grow_memory, X->getType()); 7671 return Builder.CreateCall(Callee, X); 7672 } 7673 7674 default: 7675 return nullptr; 7676 } 7677 } 7678