1 //===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This contains code to emit Builtin calls as LLVM code. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "CodeGenFunction.h" 15 #include "CGObjCRuntime.h" 16 #include "CodeGenModule.h" 17 #include "TargetInfo.h" 18 #include "clang/AST/ASTContext.h" 19 #include "clang/AST/Decl.h" 20 #include "clang/Basic/TargetBuiltins.h" 21 #include "clang/Basic/TargetInfo.h" 22 #include "clang/CodeGen/CGFunctionInfo.h" 23 #include "llvm/IR/DataLayout.h" 24 #include "llvm/IR/Intrinsics.h" 25 26 using namespace clang; 27 using namespace CodeGen; 28 using namespace llvm; 29 30 /// getBuiltinLibFunction - Given a builtin id for a function like 31 /// "__builtin_fabsf", return a Function* for "fabsf". 32 llvm::Value *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD, 33 unsigned BuiltinID) { 34 assert(Context.BuiltinInfo.isLibFunction(BuiltinID)); 35 36 // Get the name, skip over the __builtin_ prefix (if necessary). 37 StringRef Name; 38 GlobalDecl D(FD); 39 40 // If the builtin has been declared explicitly with an assembler label, 41 // use the mangled name. This differs from the plain label on platforms 42 // that prefix labels. 43 if (FD->hasAttr<AsmLabelAttr>()) 44 Name = getMangledName(D); 45 else 46 Name = Context.BuiltinInfo.GetName(BuiltinID) + 10; 47 48 llvm::FunctionType *Ty = 49 cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType())); 50 51 return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false); 52 } 53 54 /// Emit the conversions required to turn the given value into an 55 /// integer of the given size. 56 static Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V, 57 QualType T, llvm::IntegerType *IntType) { 58 V = CGF.EmitToMemory(V, T); 59 60 if (V->getType()->isPointerTy()) 61 return CGF.Builder.CreatePtrToInt(V, IntType); 62 63 assert(V->getType() == IntType); 64 return V; 65 } 66 67 static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, 68 QualType T, llvm::Type *ResultType) { 69 V = CGF.EmitFromMemory(V, T); 70 71 if (ResultType->isPointerTy()) 72 return CGF.Builder.CreateIntToPtr(V, ResultType); 73 74 assert(V->getType() == ResultType); 75 return V; 76 } 77 78 /// Utility to insert an atomic instruction based on Instrinsic::ID 79 /// and the expression node. 80 static RValue EmitBinaryAtomic(CodeGenFunction &CGF, 81 llvm::AtomicRMWInst::BinOp Kind, 82 const CallExpr *E) { 83 QualType T = E->getType(); 84 assert(E->getArg(0)->getType()->isPointerType()); 85 assert(CGF.getContext().hasSameUnqualifiedType(T, 86 E->getArg(0)->getType()->getPointeeType())); 87 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType())); 88 89 llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); 90 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 91 92 llvm::IntegerType *IntType = 93 llvm::IntegerType::get(CGF.getLLVMContext(), 94 CGF.getContext().getTypeSize(T)); 95 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 96 97 llvm::Value *Args[2]; 98 Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); 99 Args[1] = CGF.EmitScalarExpr(E->getArg(1)); 100 llvm::Type *ValueType = Args[1]->getType(); 101 Args[1] = EmitToInt(CGF, Args[1], T, IntType); 102 103 llvm::Value *Result = 104 CGF.Builder.CreateAtomicRMW(Kind, Args[0], Args[1], 105 llvm::SequentiallyConsistent); 106 Result = EmitFromInt(CGF, Result, T, ValueType); 107 return RValue::get(Result); 108 } 109 110 /// Utility to insert an atomic instruction based Instrinsic::ID and 111 /// the expression node, where the return value is the result of the 112 /// operation. 113 static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, 114 llvm::AtomicRMWInst::BinOp Kind, 115 const CallExpr *E, 116 Instruction::BinaryOps Op) { 117 QualType T = E->getType(); 118 assert(E->getArg(0)->getType()->isPointerType()); 119 assert(CGF.getContext().hasSameUnqualifiedType(T, 120 E->getArg(0)->getType()->getPointeeType())); 121 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType())); 122 123 llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); 124 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 125 126 llvm::IntegerType *IntType = 127 llvm::IntegerType::get(CGF.getLLVMContext(), 128 CGF.getContext().getTypeSize(T)); 129 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 130 131 llvm::Value *Args[2]; 132 Args[1] = CGF.EmitScalarExpr(E->getArg(1)); 133 llvm::Type *ValueType = Args[1]->getType(); 134 Args[1] = EmitToInt(CGF, Args[1], T, IntType); 135 Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); 136 137 llvm::Value *Result = 138 CGF.Builder.CreateAtomicRMW(Kind, Args[0], Args[1], 139 llvm::SequentiallyConsistent); 140 Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]); 141 Result = EmitFromInt(CGF, Result, T, ValueType); 142 return RValue::get(Result); 143 } 144 145 /// EmitFAbs - Emit a call to @llvm.fabs(). 146 static Value *EmitFAbs(CodeGenFunction &CGF, Value *V, QualType ValTy) { 147 Value *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType()); 148 llvm::CallInst *Call = CGF.Builder.CreateCall(F, V); 149 Call->setDoesNotAccessMemory(); 150 return Call; 151 } 152 153 static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *Fn, 154 const CallExpr *E, llvm::Value *calleeValue) { 155 return CGF.EmitCall(E->getCallee()->getType(), calleeValue, E, 156 ReturnValueSlot(), Fn); 157 } 158 159 /// \brief Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.* 160 /// depending on IntrinsicID. 161 /// 162 /// \arg CGF The current codegen function. 163 /// \arg IntrinsicID The ID for the Intrinsic we wish to generate. 164 /// \arg X The first argument to the llvm.*.with.overflow.*. 165 /// \arg Y The second argument to the llvm.*.with.overflow.*. 166 /// \arg Carry The carry returned by the llvm.*.with.overflow.*. 167 /// \returns The result (i.e. sum/product) returned by the intrinsic. 168 static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF, 169 const llvm::Intrinsic::ID IntrinsicID, 170 llvm::Value *X, llvm::Value *Y, 171 llvm::Value *&Carry) { 172 // Make sure we have integers of the same width. 173 assert(X->getType() == Y->getType() && 174 "Arguments must be the same type. (Did you forget to make sure both " 175 "arguments have the same integer width?)"); 176 177 llvm::Value *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType()); 178 llvm::Value *Tmp = CGF.Builder.CreateCall2(Callee, X, Y); 179 Carry = CGF.Builder.CreateExtractValue(Tmp, 1); 180 return CGF.Builder.CreateExtractValue(Tmp, 0); 181 } 182 183 RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, 184 unsigned BuiltinID, const CallExpr *E) { 185 // See if we can constant fold this builtin. If so, don't emit it at all. 186 Expr::EvalResult Result; 187 if (E->EvaluateAsRValue(Result, CGM.getContext()) && 188 !Result.hasSideEffects()) { 189 if (Result.Val.isInt()) 190 return RValue::get(llvm::ConstantInt::get(getLLVMContext(), 191 Result.Val.getInt())); 192 if (Result.Val.isFloat()) 193 return RValue::get(llvm::ConstantFP::get(getLLVMContext(), 194 Result.Val.getFloat())); 195 } 196 197 switch (BuiltinID) { 198 default: break; // Handle intrinsics and libm functions below. 199 case Builtin::BI__builtin___CFStringMakeConstantString: 200 case Builtin::BI__builtin___NSStringMakeConstantString: 201 return RValue::get(CGM.EmitConstantExpr(E, E->getType(), nullptr)); 202 case Builtin::BI__builtin_stdarg_start: 203 case Builtin::BI__builtin_va_start: 204 case Builtin::BI__va_start: 205 case Builtin::BI__builtin_va_end: { 206 Value *ArgValue = (BuiltinID == Builtin::BI__va_start) 207 ? EmitScalarExpr(E->getArg(0)) 208 : EmitVAListRef(E->getArg(0)); 209 llvm::Type *DestType = Int8PtrTy; 210 if (ArgValue->getType() != DestType) 211 ArgValue = Builder.CreateBitCast(ArgValue, DestType, 212 ArgValue->getName().data()); 213 214 Intrinsic::ID inst = (BuiltinID == Builtin::BI__builtin_va_end) ? 215 Intrinsic::vaend : Intrinsic::vastart; 216 return RValue::get(Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue)); 217 } 218 case Builtin::BI__builtin_va_copy: { 219 Value *DstPtr = EmitVAListRef(E->getArg(0)); 220 Value *SrcPtr = EmitVAListRef(E->getArg(1)); 221 222 llvm::Type *Type = Int8PtrTy; 223 224 DstPtr = Builder.CreateBitCast(DstPtr, Type); 225 SrcPtr = Builder.CreateBitCast(SrcPtr, Type); 226 return RValue::get(Builder.CreateCall2(CGM.getIntrinsic(Intrinsic::vacopy), 227 DstPtr, SrcPtr)); 228 } 229 case Builtin::BI__builtin_abs: 230 case Builtin::BI__builtin_labs: 231 case Builtin::BI__builtin_llabs: { 232 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 233 234 Value *NegOp = Builder.CreateNeg(ArgValue, "neg"); 235 Value *CmpResult = 236 Builder.CreateICmpSGE(ArgValue, 237 llvm::Constant::getNullValue(ArgValue->getType()), 238 "abscond"); 239 Value *Result = 240 Builder.CreateSelect(CmpResult, ArgValue, NegOp, "abs"); 241 242 return RValue::get(Result); 243 } 244 245 case Builtin::BI__builtin_conj: 246 case Builtin::BI__builtin_conjf: 247 case Builtin::BI__builtin_conjl: { 248 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 249 Value *Real = ComplexVal.first; 250 Value *Imag = ComplexVal.second; 251 Value *Zero = 252 Imag->getType()->isFPOrFPVectorTy() 253 ? llvm::ConstantFP::getZeroValueForNegation(Imag->getType()) 254 : llvm::Constant::getNullValue(Imag->getType()); 255 256 Imag = Builder.CreateFSub(Zero, Imag, "sub"); 257 return RValue::getComplex(std::make_pair(Real, Imag)); 258 } 259 case Builtin::BI__builtin_creal: 260 case Builtin::BI__builtin_crealf: 261 case Builtin::BI__builtin_creall: 262 case Builtin::BIcreal: 263 case Builtin::BIcrealf: 264 case Builtin::BIcreall: { 265 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 266 return RValue::get(ComplexVal.first); 267 } 268 269 case Builtin::BI__builtin_cimag: 270 case Builtin::BI__builtin_cimagf: 271 case Builtin::BI__builtin_cimagl: 272 case Builtin::BIcimag: 273 case Builtin::BIcimagf: 274 case Builtin::BIcimagl: { 275 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 276 return RValue::get(ComplexVal.second); 277 } 278 279 case Builtin::BI__builtin_ctzs: 280 case Builtin::BI__builtin_ctz: 281 case Builtin::BI__builtin_ctzl: 282 case Builtin::BI__builtin_ctzll: { 283 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 284 285 llvm::Type *ArgType = ArgValue->getType(); 286 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); 287 288 llvm::Type *ResultType = ConvertType(E->getType()); 289 Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); 290 Value *Result = Builder.CreateCall2(F, ArgValue, ZeroUndef); 291 if (Result->getType() != ResultType) 292 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 293 "cast"); 294 return RValue::get(Result); 295 } 296 case Builtin::BI__builtin_clzs: 297 case Builtin::BI__builtin_clz: 298 case Builtin::BI__builtin_clzl: 299 case Builtin::BI__builtin_clzll: { 300 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 301 302 llvm::Type *ArgType = ArgValue->getType(); 303 Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType); 304 305 llvm::Type *ResultType = ConvertType(E->getType()); 306 Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); 307 Value *Result = Builder.CreateCall2(F, ArgValue, ZeroUndef); 308 if (Result->getType() != ResultType) 309 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 310 "cast"); 311 return RValue::get(Result); 312 } 313 case Builtin::BI__builtin_ffs: 314 case Builtin::BI__builtin_ffsl: 315 case Builtin::BI__builtin_ffsll: { 316 // ffs(x) -> x ? cttz(x) + 1 : 0 317 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 318 319 llvm::Type *ArgType = ArgValue->getType(); 320 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); 321 322 llvm::Type *ResultType = ConvertType(E->getType()); 323 Value *Tmp = Builder.CreateAdd(Builder.CreateCall2(F, ArgValue, 324 Builder.getTrue()), 325 llvm::ConstantInt::get(ArgType, 1)); 326 Value *Zero = llvm::Constant::getNullValue(ArgType); 327 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero"); 328 Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs"); 329 if (Result->getType() != ResultType) 330 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 331 "cast"); 332 return RValue::get(Result); 333 } 334 case Builtin::BI__builtin_parity: 335 case Builtin::BI__builtin_parityl: 336 case Builtin::BI__builtin_parityll: { 337 // parity(x) -> ctpop(x) & 1 338 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 339 340 llvm::Type *ArgType = ArgValue->getType(); 341 Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType); 342 343 llvm::Type *ResultType = ConvertType(E->getType()); 344 Value *Tmp = Builder.CreateCall(F, ArgValue); 345 Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1)); 346 if (Result->getType() != ResultType) 347 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 348 "cast"); 349 return RValue::get(Result); 350 } 351 case Builtin::BI__builtin_popcount: 352 case Builtin::BI__builtin_popcountl: 353 case Builtin::BI__builtin_popcountll: { 354 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 355 356 llvm::Type *ArgType = ArgValue->getType(); 357 Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType); 358 359 llvm::Type *ResultType = ConvertType(E->getType()); 360 Value *Result = Builder.CreateCall(F, ArgValue); 361 if (Result->getType() != ResultType) 362 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 363 "cast"); 364 return RValue::get(Result); 365 } 366 case Builtin::BI__builtin_expect: { 367 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 368 llvm::Type *ArgType = ArgValue->getType(); 369 370 Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType); 371 Value *ExpectedValue = EmitScalarExpr(E->getArg(1)); 372 373 Value *Result = Builder.CreateCall2(FnExpect, ArgValue, ExpectedValue, 374 "expval"); 375 return RValue::get(Result); 376 } 377 case Builtin::BI__builtin_assume_aligned: { 378 Value *PtrValue = EmitScalarExpr(E->getArg(0)); 379 Value *OffsetValue = 380 (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : nullptr; 381 382 Value *AlignmentValue = EmitScalarExpr(E->getArg(1)); 383 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue); 384 unsigned Alignment = (unsigned) AlignmentCI->getZExtValue(); 385 386 EmitAlignmentAssumption(PtrValue, Alignment, OffsetValue); 387 return RValue::get(PtrValue); 388 } 389 case Builtin::BI__assume: 390 case Builtin::BI__builtin_assume: { 391 if (E->getArg(0)->HasSideEffects(getContext())) 392 return RValue::get(nullptr); 393 394 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 395 Value *FnAssume = CGM.getIntrinsic(Intrinsic::assume); 396 return RValue::get(Builder.CreateCall(FnAssume, ArgValue)); 397 } 398 case Builtin::BI__builtin_bswap16: 399 case Builtin::BI__builtin_bswap32: 400 case Builtin::BI__builtin_bswap64: { 401 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 402 llvm::Type *ArgType = ArgValue->getType(); 403 Value *F = CGM.getIntrinsic(Intrinsic::bswap, ArgType); 404 return RValue::get(Builder.CreateCall(F, ArgValue)); 405 } 406 case Builtin::BI__builtin_object_size: { 407 // We rely on constant folding to deal with expressions with side effects. 408 assert(!E->getArg(0)->HasSideEffects(getContext()) && 409 "should have been constant folded"); 410 411 // We pass this builtin onto the optimizer so that it can 412 // figure out the object size in more complex cases. 413 llvm::Type *ResType = ConvertType(E->getType()); 414 415 // LLVM only supports 0 and 2, make sure that we pass along that 416 // as a boolean. 417 Value *Ty = EmitScalarExpr(E->getArg(1)); 418 ConstantInt *CI = dyn_cast<ConstantInt>(Ty); 419 assert(CI); 420 uint64_t val = CI->getZExtValue(); 421 CI = ConstantInt::get(Builder.getInt1Ty(), (val & 0x2) >> 1); 422 // FIXME: Get right address space. 423 llvm::Type *Tys[] = { ResType, Builder.getInt8PtrTy(0) }; 424 Value *F = CGM.getIntrinsic(Intrinsic::objectsize, Tys); 425 return RValue::get(Builder.CreateCall2(F, EmitScalarExpr(E->getArg(0)),CI)); 426 } 427 case Builtin::BI__builtin_prefetch: { 428 Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0)); 429 // FIXME: Technically these constants should of type 'int', yes? 430 RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) : 431 llvm::ConstantInt::get(Int32Ty, 0); 432 Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : 433 llvm::ConstantInt::get(Int32Ty, 3); 434 Value *Data = llvm::ConstantInt::get(Int32Ty, 1); 435 Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 436 return RValue::get(Builder.CreateCall4(F, Address, RW, Locality, Data)); 437 } 438 case Builtin::BI__builtin_readcyclecounter: { 439 Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter); 440 return RValue::get(Builder.CreateCall(F)); 441 } 442 case Builtin::BI__builtin___clear_cache: { 443 Value *Begin = EmitScalarExpr(E->getArg(0)); 444 Value *End = EmitScalarExpr(E->getArg(1)); 445 Value *F = CGM.getIntrinsic(Intrinsic::clear_cache); 446 return RValue::get(Builder.CreateCall2(F, Begin, End)); 447 } 448 case Builtin::BI__builtin_trap: { 449 Value *F = CGM.getIntrinsic(Intrinsic::trap); 450 return RValue::get(Builder.CreateCall(F)); 451 } 452 case Builtin::BI__debugbreak: { 453 Value *F = CGM.getIntrinsic(Intrinsic::debugtrap); 454 return RValue::get(Builder.CreateCall(F)); 455 } 456 case Builtin::BI__builtin_unreachable: { 457 if (SanOpts->Unreachable) { 458 SanitizerScope SanScope(this); 459 EmitCheck(Builder.getFalse(), "builtin_unreachable", 460 EmitCheckSourceLocation(E->getExprLoc()), 461 None, CRK_Unrecoverable); 462 } else 463 Builder.CreateUnreachable(); 464 465 // We do need to preserve an insertion point. 466 EmitBlock(createBasicBlock("unreachable.cont")); 467 468 return RValue::get(nullptr); 469 } 470 471 case Builtin::BI__builtin_powi: 472 case Builtin::BI__builtin_powif: 473 case Builtin::BI__builtin_powil: { 474 Value *Base = EmitScalarExpr(E->getArg(0)); 475 Value *Exponent = EmitScalarExpr(E->getArg(1)); 476 llvm::Type *ArgType = Base->getType(); 477 Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType); 478 return RValue::get(Builder.CreateCall2(F, Base, Exponent)); 479 } 480 481 case Builtin::BI__builtin_isgreater: 482 case Builtin::BI__builtin_isgreaterequal: 483 case Builtin::BI__builtin_isless: 484 case Builtin::BI__builtin_islessequal: 485 case Builtin::BI__builtin_islessgreater: 486 case Builtin::BI__builtin_isunordered: { 487 // Ordered comparisons: we know the arguments to these are matching scalar 488 // floating point values. 489 Value *LHS = EmitScalarExpr(E->getArg(0)); 490 Value *RHS = EmitScalarExpr(E->getArg(1)); 491 492 switch (BuiltinID) { 493 default: llvm_unreachable("Unknown ordered comparison"); 494 case Builtin::BI__builtin_isgreater: 495 LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp"); 496 break; 497 case Builtin::BI__builtin_isgreaterequal: 498 LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp"); 499 break; 500 case Builtin::BI__builtin_isless: 501 LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp"); 502 break; 503 case Builtin::BI__builtin_islessequal: 504 LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp"); 505 break; 506 case Builtin::BI__builtin_islessgreater: 507 LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp"); 508 break; 509 case Builtin::BI__builtin_isunordered: 510 LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp"); 511 break; 512 } 513 // ZExt bool to int type. 514 return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType()))); 515 } 516 case Builtin::BI__builtin_isnan: { 517 Value *V = EmitScalarExpr(E->getArg(0)); 518 V = Builder.CreateFCmpUNO(V, V, "cmp"); 519 return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); 520 } 521 522 case Builtin::BI__builtin_isinf: { 523 // isinf(x) --> fabs(x) == infinity 524 Value *V = EmitScalarExpr(E->getArg(0)); 525 V = EmitFAbs(*this, V, E->getArg(0)->getType()); 526 527 V = Builder.CreateFCmpOEQ(V, ConstantFP::getInfinity(V->getType()),"isinf"); 528 return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); 529 } 530 531 // TODO: BI__builtin_isinf_sign 532 // isinf_sign(x) -> isinf(x) ? (signbit(x) ? -1 : 1) : 0 533 534 case Builtin::BI__builtin_isnormal: { 535 // isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min 536 Value *V = EmitScalarExpr(E->getArg(0)); 537 Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq"); 538 539 Value *Abs = EmitFAbs(*this, V, E->getArg(0)->getType()); 540 Value *IsLessThanInf = 541 Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),"isinf"); 542 APFloat Smallest = APFloat::getSmallestNormalized( 543 getContext().getFloatTypeSemantics(E->getArg(0)->getType())); 544 Value *IsNormal = 545 Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest), 546 "isnormal"); 547 V = Builder.CreateAnd(Eq, IsLessThanInf, "and"); 548 V = Builder.CreateAnd(V, IsNormal, "and"); 549 return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); 550 } 551 552 case Builtin::BI__builtin_isfinite: { 553 // isfinite(x) --> x == x && fabs(x) != infinity; 554 Value *V = EmitScalarExpr(E->getArg(0)); 555 Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq"); 556 557 Value *Abs = EmitFAbs(*this, V, E->getArg(0)->getType()); 558 Value *IsNotInf = 559 Builder.CreateFCmpUNE(Abs, ConstantFP::getInfinity(V->getType()),"isinf"); 560 561 V = Builder.CreateAnd(Eq, IsNotInf, "and"); 562 return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); 563 } 564 565 case Builtin::BI__builtin_fpclassify: { 566 Value *V = EmitScalarExpr(E->getArg(5)); 567 llvm::Type *Ty = ConvertType(E->getArg(5)->getType()); 568 569 // Create Result 570 BasicBlock *Begin = Builder.GetInsertBlock(); 571 BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn); 572 Builder.SetInsertPoint(End); 573 PHINode *Result = 574 Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4, 575 "fpclassify_result"); 576 577 // if (V==0) return FP_ZERO 578 Builder.SetInsertPoint(Begin); 579 Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty), 580 "iszero"); 581 Value *ZeroLiteral = EmitScalarExpr(E->getArg(4)); 582 BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn); 583 Builder.CreateCondBr(IsZero, End, NotZero); 584 Result->addIncoming(ZeroLiteral, Begin); 585 586 // if (V != V) return FP_NAN 587 Builder.SetInsertPoint(NotZero); 588 Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp"); 589 Value *NanLiteral = EmitScalarExpr(E->getArg(0)); 590 BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn); 591 Builder.CreateCondBr(IsNan, End, NotNan); 592 Result->addIncoming(NanLiteral, NotZero); 593 594 // if (fabs(V) == infinity) return FP_INFINITY 595 Builder.SetInsertPoint(NotNan); 596 Value *VAbs = EmitFAbs(*this, V, E->getArg(5)->getType()); 597 Value *IsInf = 598 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()), 599 "isinf"); 600 Value *InfLiteral = EmitScalarExpr(E->getArg(1)); 601 BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn); 602 Builder.CreateCondBr(IsInf, End, NotInf); 603 Result->addIncoming(InfLiteral, NotNan); 604 605 // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL 606 Builder.SetInsertPoint(NotInf); 607 APFloat Smallest = APFloat::getSmallestNormalized( 608 getContext().getFloatTypeSemantics(E->getArg(5)->getType())); 609 Value *IsNormal = 610 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest), 611 "isnormal"); 612 Value *NormalResult = 613 Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)), 614 EmitScalarExpr(E->getArg(3))); 615 Builder.CreateBr(End); 616 Result->addIncoming(NormalResult, NotInf); 617 618 // return Result 619 Builder.SetInsertPoint(End); 620 return RValue::get(Result); 621 } 622 623 case Builtin::BIalloca: 624 case Builtin::BI_alloca: 625 case Builtin::BI__builtin_alloca: { 626 Value *Size = EmitScalarExpr(E->getArg(0)); 627 return RValue::get(Builder.CreateAlloca(Builder.getInt8Ty(), Size)); 628 } 629 case Builtin::BIbzero: 630 case Builtin::BI__builtin_bzero: { 631 std::pair<llvm::Value*, unsigned> Dest = 632 EmitPointerWithAlignment(E->getArg(0)); 633 Value *SizeVal = EmitScalarExpr(E->getArg(1)); 634 Builder.CreateMemSet(Dest.first, Builder.getInt8(0), SizeVal, 635 Dest.second, false); 636 return RValue::get(Dest.first); 637 } 638 case Builtin::BImemcpy: 639 case Builtin::BI__builtin_memcpy: { 640 std::pair<llvm::Value*, unsigned> Dest = 641 EmitPointerWithAlignment(E->getArg(0)); 642 std::pair<llvm::Value*, unsigned> Src = 643 EmitPointerWithAlignment(E->getArg(1)); 644 Value *SizeVal = EmitScalarExpr(E->getArg(2)); 645 unsigned Align = std::min(Dest.second, Src.second); 646 Builder.CreateMemCpy(Dest.first, Src.first, SizeVal, Align, false); 647 return RValue::get(Dest.first); 648 } 649 650 case Builtin::BI__builtin___memcpy_chk: { 651 // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2. 652 llvm::APSInt Size, DstSize; 653 if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 654 !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 655 break; 656 if (Size.ugt(DstSize)) 657 break; 658 std::pair<llvm::Value*, unsigned> Dest = 659 EmitPointerWithAlignment(E->getArg(0)); 660 std::pair<llvm::Value*, unsigned> Src = 661 EmitPointerWithAlignment(E->getArg(1)); 662 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 663 unsigned Align = std::min(Dest.second, Src.second); 664 Builder.CreateMemCpy(Dest.first, Src.first, SizeVal, Align, false); 665 return RValue::get(Dest.first); 666 } 667 668 case Builtin::BI__builtin_objc_memmove_collectable: { 669 Value *Address = EmitScalarExpr(E->getArg(0)); 670 Value *SrcAddr = EmitScalarExpr(E->getArg(1)); 671 Value *SizeVal = EmitScalarExpr(E->getArg(2)); 672 CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, 673 Address, SrcAddr, SizeVal); 674 return RValue::get(Address); 675 } 676 677 case Builtin::BI__builtin___memmove_chk: { 678 // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2. 679 llvm::APSInt Size, DstSize; 680 if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 681 !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 682 break; 683 if (Size.ugt(DstSize)) 684 break; 685 std::pair<llvm::Value*, unsigned> Dest = 686 EmitPointerWithAlignment(E->getArg(0)); 687 std::pair<llvm::Value*, unsigned> Src = 688 EmitPointerWithAlignment(E->getArg(1)); 689 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 690 unsigned Align = std::min(Dest.second, Src.second); 691 Builder.CreateMemMove(Dest.first, Src.first, SizeVal, Align, false); 692 return RValue::get(Dest.first); 693 } 694 695 case Builtin::BImemmove: 696 case Builtin::BI__builtin_memmove: { 697 std::pair<llvm::Value*, unsigned> Dest = 698 EmitPointerWithAlignment(E->getArg(0)); 699 std::pair<llvm::Value*, unsigned> Src = 700 EmitPointerWithAlignment(E->getArg(1)); 701 Value *SizeVal = EmitScalarExpr(E->getArg(2)); 702 unsigned Align = std::min(Dest.second, Src.second); 703 Builder.CreateMemMove(Dest.first, Src.first, SizeVal, Align, false); 704 return RValue::get(Dest.first); 705 } 706 case Builtin::BImemset: 707 case Builtin::BI__builtin_memset: { 708 std::pair<llvm::Value*, unsigned> Dest = 709 EmitPointerWithAlignment(E->getArg(0)); 710 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), 711 Builder.getInt8Ty()); 712 Value *SizeVal = EmitScalarExpr(E->getArg(2)); 713 Builder.CreateMemSet(Dest.first, ByteVal, SizeVal, Dest.second, false); 714 return RValue::get(Dest.first); 715 } 716 case Builtin::BI__builtin___memset_chk: { 717 // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2. 718 llvm::APSInt Size, DstSize; 719 if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 720 !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 721 break; 722 if (Size.ugt(DstSize)) 723 break; 724 std::pair<llvm::Value*, unsigned> Dest = 725 EmitPointerWithAlignment(E->getArg(0)); 726 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), 727 Builder.getInt8Ty()); 728 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 729 Builder.CreateMemSet(Dest.first, ByteVal, SizeVal, Dest.second, false); 730 return RValue::get(Dest.first); 731 } 732 case Builtin::BI__builtin_dwarf_cfa: { 733 // The offset in bytes from the first argument to the CFA. 734 // 735 // Why on earth is this in the frontend? Is there any reason at 736 // all that the backend can't reasonably determine this while 737 // lowering llvm.eh.dwarf.cfa()? 738 // 739 // TODO: If there's a satisfactory reason, add a target hook for 740 // this instead of hard-coding 0, which is correct for most targets. 741 int32_t Offset = 0; 742 743 Value *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa); 744 return RValue::get(Builder.CreateCall(F, 745 llvm::ConstantInt::get(Int32Ty, Offset))); 746 } 747 case Builtin::BI__builtin_return_address: { 748 Value *Depth = EmitScalarExpr(E->getArg(0)); 749 Depth = Builder.CreateIntCast(Depth, Int32Ty, false); 750 Value *F = CGM.getIntrinsic(Intrinsic::returnaddress); 751 return RValue::get(Builder.CreateCall(F, Depth)); 752 } 753 case Builtin::BI__builtin_frame_address: { 754 Value *Depth = EmitScalarExpr(E->getArg(0)); 755 Depth = Builder.CreateIntCast(Depth, Int32Ty, false); 756 Value *F = CGM.getIntrinsic(Intrinsic::frameaddress); 757 return RValue::get(Builder.CreateCall(F, Depth)); 758 } 759 case Builtin::BI__builtin_extract_return_addr: { 760 Value *Address = EmitScalarExpr(E->getArg(0)); 761 Value *Result = getTargetHooks().decodeReturnAddress(*this, Address); 762 return RValue::get(Result); 763 } 764 case Builtin::BI__builtin_frob_return_addr: { 765 Value *Address = EmitScalarExpr(E->getArg(0)); 766 Value *Result = getTargetHooks().encodeReturnAddress(*this, Address); 767 return RValue::get(Result); 768 } 769 case Builtin::BI__builtin_dwarf_sp_column: { 770 llvm::IntegerType *Ty 771 = cast<llvm::IntegerType>(ConvertType(E->getType())); 772 int Column = getTargetHooks().getDwarfEHStackPointer(CGM); 773 if (Column == -1) { 774 CGM.ErrorUnsupported(E, "__builtin_dwarf_sp_column"); 775 return RValue::get(llvm::UndefValue::get(Ty)); 776 } 777 return RValue::get(llvm::ConstantInt::get(Ty, Column, true)); 778 } 779 case Builtin::BI__builtin_init_dwarf_reg_size_table: { 780 Value *Address = EmitScalarExpr(E->getArg(0)); 781 if (getTargetHooks().initDwarfEHRegSizeTable(*this, Address)) 782 CGM.ErrorUnsupported(E, "__builtin_init_dwarf_reg_size_table"); 783 return RValue::get(llvm::UndefValue::get(ConvertType(E->getType()))); 784 } 785 case Builtin::BI__builtin_eh_return: { 786 Value *Int = EmitScalarExpr(E->getArg(0)); 787 Value *Ptr = EmitScalarExpr(E->getArg(1)); 788 789 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType()); 790 assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) && 791 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants"); 792 Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32 793 ? Intrinsic::eh_return_i32 794 : Intrinsic::eh_return_i64); 795 Builder.CreateCall2(F, Int, Ptr); 796 Builder.CreateUnreachable(); 797 798 // We do need to preserve an insertion point. 799 EmitBlock(createBasicBlock("builtin_eh_return.cont")); 800 801 return RValue::get(nullptr); 802 } 803 case Builtin::BI__builtin_unwind_init: { 804 Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init); 805 return RValue::get(Builder.CreateCall(F)); 806 } 807 case Builtin::BI__builtin_extend_pointer: { 808 // Extends a pointer to the size of an _Unwind_Word, which is 809 // uint64_t on all platforms. Generally this gets poked into a 810 // register and eventually used as an address, so if the 811 // addressing registers are wider than pointers and the platform 812 // doesn't implicitly ignore high-order bits when doing 813 // addressing, we need to make sure we zext / sext based on 814 // the platform's expectations. 815 // 816 // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html 817 818 // Cast the pointer to intptr_t. 819 Value *Ptr = EmitScalarExpr(E->getArg(0)); 820 Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast"); 821 822 // If that's 64 bits, we're done. 823 if (IntPtrTy->getBitWidth() == 64) 824 return RValue::get(Result); 825 826 // Otherwise, ask the codegen data what to do. 827 if (getTargetHooks().extendPointerWithSExt()) 828 return RValue::get(Builder.CreateSExt(Result, Int64Ty, "extend.sext")); 829 else 830 return RValue::get(Builder.CreateZExt(Result, Int64Ty, "extend.zext")); 831 } 832 case Builtin::BI__builtin_setjmp: { 833 // Buffer is a void**. 834 Value *Buf = EmitScalarExpr(E->getArg(0)); 835 836 // Store the frame pointer to the setjmp buffer. 837 Value *FrameAddr = 838 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), 839 ConstantInt::get(Int32Ty, 0)); 840 Builder.CreateStore(FrameAddr, Buf); 841 842 // Store the stack pointer to the setjmp buffer. 843 Value *StackAddr = 844 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave)); 845 Value *StackSaveSlot = 846 Builder.CreateGEP(Buf, ConstantInt::get(Int32Ty, 2)); 847 Builder.CreateStore(StackAddr, StackSaveSlot); 848 849 // Call LLVM's EH setjmp, which is lightweight. 850 Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp); 851 Buf = Builder.CreateBitCast(Buf, Int8PtrTy); 852 return RValue::get(Builder.CreateCall(F, Buf)); 853 } 854 case Builtin::BI__builtin_longjmp: { 855 Value *Buf = EmitScalarExpr(E->getArg(0)); 856 Buf = Builder.CreateBitCast(Buf, Int8PtrTy); 857 858 // Call LLVM's EH longjmp, which is lightweight. 859 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf); 860 861 // longjmp doesn't return; mark this as unreachable. 862 Builder.CreateUnreachable(); 863 864 // We do need to preserve an insertion point. 865 EmitBlock(createBasicBlock("longjmp.cont")); 866 867 return RValue::get(nullptr); 868 } 869 case Builtin::BI__sync_fetch_and_add: 870 case Builtin::BI__sync_fetch_and_sub: 871 case Builtin::BI__sync_fetch_and_or: 872 case Builtin::BI__sync_fetch_and_and: 873 case Builtin::BI__sync_fetch_and_xor: 874 case Builtin::BI__sync_add_and_fetch: 875 case Builtin::BI__sync_sub_and_fetch: 876 case Builtin::BI__sync_and_and_fetch: 877 case Builtin::BI__sync_or_and_fetch: 878 case Builtin::BI__sync_xor_and_fetch: 879 case Builtin::BI__sync_val_compare_and_swap: 880 case Builtin::BI__sync_bool_compare_and_swap: 881 case Builtin::BI__sync_lock_test_and_set: 882 case Builtin::BI__sync_lock_release: 883 case Builtin::BI__sync_swap: 884 llvm_unreachable("Shouldn't make it through sema"); 885 case Builtin::BI__sync_fetch_and_add_1: 886 case Builtin::BI__sync_fetch_and_add_2: 887 case Builtin::BI__sync_fetch_and_add_4: 888 case Builtin::BI__sync_fetch_and_add_8: 889 case Builtin::BI__sync_fetch_and_add_16: 890 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Add, E); 891 case Builtin::BI__sync_fetch_and_sub_1: 892 case Builtin::BI__sync_fetch_and_sub_2: 893 case Builtin::BI__sync_fetch_and_sub_4: 894 case Builtin::BI__sync_fetch_and_sub_8: 895 case Builtin::BI__sync_fetch_and_sub_16: 896 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Sub, E); 897 case Builtin::BI__sync_fetch_and_or_1: 898 case Builtin::BI__sync_fetch_and_or_2: 899 case Builtin::BI__sync_fetch_and_or_4: 900 case Builtin::BI__sync_fetch_and_or_8: 901 case Builtin::BI__sync_fetch_and_or_16: 902 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Or, E); 903 case Builtin::BI__sync_fetch_and_and_1: 904 case Builtin::BI__sync_fetch_and_and_2: 905 case Builtin::BI__sync_fetch_and_and_4: 906 case Builtin::BI__sync_fetch_and_and_8: 907 case Builtin::BI__sync_fetch_and_and_16: 908 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::And, E); 909 case Builtin::BI__sync_fetch_and_xor_1: 910 case Builtin::BI__sync_fetch_and_xor_2: 911 case Builtin::BI__sync_fetch_and_xor_4: 912 case Builtin::BI__sync_fetch_and_xor_8: 913 case Builtin::BI__sync_fetch_and_xor_16: 914 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xor, E); 915 916 // Clang extensions: not overloaded yet. 917 case Builtin::BI__sync_fetch_and_min: 918 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Min, E); 919 case Builtin::BI__sync_fetch_and_max: 920 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Max, E); 921 case Builtin::BI__sync_fetch_and_umin: 922 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMin, E); 923 case Builtin::BI__sync_fetch_and_umax: 924 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMax, E); 925 926 case Builtin::BI__sync_add_and_fetch_1: 927 case Builtin::BI__sync_add_and_fetch_2: 928 case Builtin::BI__sync_add_and_fetch_4: 929 case Builtin::BI__sync_add_and_fetch_8: 930 case Builtin::BI__sync_add_and_fetch_16: 931 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Add, E, 932 llvm::Instruction::Add); 933 case Builtin::BI__sync_sub_and_fetch_1: 934 case Builtin::BI__sync_sub_and_fetch_2: 935 case Builtin::BI__sync_sub_and_fetch_4: 936 case Builtin::BI__sync_sub_and_fetch_8: 937 case Builtin::BI__sync_sub_and_fetch_16: 938 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Sub, E, 939 llvm::Instruction::Sub); 940 case Builtin::BI__sync_and_and_fetch_1: 941 case Builtin::BI__sync_and_and_fetch_2: 942 case Builtin::BI__sync_and_and_fetch_4: 943 case Builtin::BI__sync_and_and_fetch_8: 944 case Builtin::BI__sync_and_and_fetch_16: 945 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::And, E, 946 llvm::Instruction::And); 947 case Builtin::BI__sync_or_and_fetch_1: 948 case Builtin::BI__sync_or_and_fetch_2: 949 case Builtin::BI__sync_or_and_fetch_4: 950 case Builtin::BI__sync_or_and_fetch_8: 951 case Builtin::BI__sync_or_and_fetch_16: 952 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Or, E, 953 llvm::Instruction::Or); 954 case Builtin::BI__sync_xor_and_fetch_1: 955 case Builtin::BI__sync_xor_and_fetch_2: 956 case Builtin::BI__sync_xor_and_fetch_4: 957 case Builtin::BI__sync_xor_and_fetch_8: 958 case Builtin::BI__sync_xor_and_fetch_16: 959 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Xor, E, 960 llvm::Instruction::Xor); 961 962 case Builtin::BI__sync_val_compare_and_swap_1: 963 case Builtin::BI__sync_val_compare_and_swap_2: 964 case Builtin::BI__sync_val_compare_and_swap_4: 965 case Builtin::BI__sync_val_compare_and_swap_8: 966 case Builtin::BI__sync_val_compare_and_swap_16: { 967 QualType T = E->getType(); 968 llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0)); 969 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 970 971 llvm::IntegerType *IntType = 972 llvm::IntegerType::get(getLLVMContext(), 973 getContext().getTypeSize(T)); 974 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 975 976 Value *Args[3]; 977 Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType); 978 Args[1] = EmitScalarExpr(E->getArg(1)); 979 llvm::Type *ValueType = Args[1]->getType(); 980 Args[1] = EmitToInt(*this, Args[1], T, IntType); 981 Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType); 982 983 Value *Result = Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2], 984 llvm::SequentiallyConsistent, 985 llvm::SequentiallyConsistent); 986 Result = Builder.CreateExtractValue(Result, 0); 987 Result = EmitFromInt(*this, Result, T, ValueType); 988 return RValue::get(Result); 989 } 990 991 case Builtin::BI__sync_bool_compare_and_swap_1: 992 case Builtin::BI__sync_bool_compare_and_swap_2: 993 case Builtin::BI__sync_bool_compare_and_swap_4: 994 case Builtin::BI__sync_bool_compare_and_swap_8: 995 case Builtin::BI__sync_bool_compare_and_swap_16: { 996 QualType T = E->getArg(1)->getType(); 997 llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0)); 998 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 999 1000 llvm::IntegerType *IntType = 1001 llvm::IntegerType::get(getLLVMContext(), 1002 getContext().getTypeSize(T)); 1003 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 1004 1005 Value *Args[3]; 1006 Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType); 1007 Args[1] = EmitToInt(*this, EmitScalarExpr(E->getArg(1)), T, IntType); 1008 Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType); 1009 1010 Value *Pair = Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2], 1011 llvm::SequentiallyConsistent, 1012 llvm::SequentiallyConsistent); 1013 Value *Result = Builder.CreateExtractValue(Pair, 1); 1014 // zext bool to int. 1015 Result = Builder.CreateZExt(Result, ConvertType(E->getType())); 1016 return RValue::get(Result); 1017 } 1018 1019 case Builtin::BI__sync_swap_1: 1020 case Builtin::BI__sync_swap_2: 1021 case Builtin::BI__sync_swap_4: 1022 case Builtin::BI__sync_swap_8: 1023 case Builtin::BI__sync_swap_16: 1024 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); 1025 1026 case Builtin::BI__sync_lock_test_and_set_1: 1027 case Builtin::BI__sync_lock_test_and_set_2: 1028 case Builtin::BI__sync_lock_test_and_set_4: 1029 case Builtin::BI__sync_lock_test_and_set_8: 1030 case Builtin::BI__sync_lock_test_and_set_16: 1031 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); 1032 1033 case Builtin::BI__sync_lock_release_1: 1034 case Builtin::BI__sync_lock_release_2: 1035 case Builtin::BI__sync_lock_release_4: 1036 case Builtin::BI__sync_lock_release_8: 1037 case Builtin::BI__sync_lock_release_16: { 1038 Value *Ptr = EmitScalarExpr(E->getArg(0)); 1039 QualType ElTy = E->getArg(0)->getType()->getPointeeType(); 1040 CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy); 1041 llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(), 1042 StoreSize.getQuantity() * 8); 1043 Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo()); 1044 llvm::StoreInst *Store = 1045 Builder.CreateStore(llvm::Constant::getNullValue(ITy), Ptr); 1046 Store->setAlignment(StoreSize.getQuantity()); 1047 Store->setAtomic(llvm::Release); 1048 return RValue::get(nullptr); 1049 } 1050 1051 case Builtin::BI__sync_synchronize: { 1052 // We assume this is supposed to correspond to a C++0x-style 1053 // sequentially-consistent fence (i.e. this is only usable for 1054 // synchonization, not device I/O or anything like that). This intrinsic 1055 // is really badly designed in the sense that in theory, there isn't 1056 // any way to safely use it... but in practice, it mostly works 1057 // to use it with non-atomic loads and stores to get acquire/release 1058 // semantics. 1059 Builder.CreateFence(llvm::SequentiallyConsistent); 1060 return RValue::get(nullptr); 1061 } 1062 1063 case Builtin::BI__c11_atomic_is_lock_free: 1064 case Builtin::BI__atomic_is_lock_free: { 1065 // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the 1066 // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since 1067 // _Atomic(T) is always properly-aligned. 1068 const char *LibCallName = "__atomic_is_lock_free"; 1069 CallArgList Args; 1070 Args.add(RValue::get(EmitScalarExpr(E->getArg(0))), 1071 getContext().getSizeType()); 1072 if (BuiltinID == Builtin::BI__atomic_is_lock_free) 1073 Args.add(RValue::get(EmitScalarExpr(E->getArg(1))), 1074 getContext().VoidPtrTy); 1075 else 1076 Args.add(RValue::get(llvm::Constant::getNullValue(VoidPtrTy)), 1077 getContext().VoidPtrTy); 1078 const CGFunctionInfo &FuncInfo = 1079 CGM.getTypes().arrangeFreeFunctionCall(E->getType(), Args, 1080 FunctionType::ExtInfo(), 1081 RequiredArgs::All); 1082 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo); 1083 llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName); 1084 return EmitCall(FuncInfo, Func, ReturnValueSlot(), Args); 1085 } 1086 1087 case Builtin::BI__atomic_test_and_set: { 1088 // Look at the argument type to determine whether this is a volatile 1089 // operation. The parameter type is always volatile. 1090 QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType(); 1091 bool Volatile = 1092 PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified(); 1093 1094 Value *Ptr = EmitScalarExpr(E->getArg(0)); 1095 unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace(); 1096 Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace)); 1097 Value *NewVal = Builder.getInt8(1); 1098 Value *Order = EmitScalarExpr(E->getArg(1)); 1099 if (isa<llvm::ConstantInt>(Order)) { 1100 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1101 AtomicRMWInst *Result = nullptr; 1102 switch (ord) { 1103 case 0: // memory_order_relaxed 1104 default: // invalid order 1105 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 1106 Ptr, NewVal, 1107 llvm::Monotonic); 1108 break; 1109 case 1: // memory_order_consume 1110 case 2: // memory_order_acquire 1111 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 1112 Ptr, NewVal, 1113 llvm::Acquire); 1114 break; 1115 case 3: // memory_order_release 1116 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 1117 Ptr, NewVal, 1118 llvm::Release); 1119 break; 1120 case 4: // memory_order_acq_rel 1121 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 1122 Ptr, NewVal, 1123 llvm::AcquireRelease); 1124 break; 1125 case 5: // memory_order_seq_cst 1126 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 1127 Ptr, NewVal, 1128 llvm::SequentiallyConsistent); 1129 break; 1130 } 1131 Result->setVolatile(Volatile); 1132 return RValue::get(Builder.CreateIsNotNull(Result, "tobool")); 1133 } 1134 1135 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1136 1137 llvm::BasicBlock *BBs[5] = { 1138 createBasicBlock("monotonic", CurFn), 1139 createBasicBlock("acquire", CurFn), 1140 createBasicBlock("release", CurFn), 1141 createBasicBlock("acqrel", CurFn), 1142 createBasicBlock("seqcst", CurFn) 1143 }; 1144 llvm::AtomicOrdering Orders[5] = { 1145 llvm::Monotonic, llvm::Acquire, llvm::Release, 1146 llvm::AcquireRelease, llvm::SequentiallyConsistent 1147 }; 1148 1149 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1150 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]); 1151 1152 Builder.SetInsertPoint(ContBB); 1153 PHINode *Result = Builder.CreatePHI(Int8Ty, 5, "was_set"); 1154 1155 for (unsigned i = 0; i < 5; ++i) { 1156 Builder.SetInsertPoint(BBs[i]); 1157 AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 1158 Ptr, NewVal, Orders[i]); 1159 RMW->setVolatile(Volatile); 1160 Result->addIncoming(RMW, BBs[i]); 1161 Builder.CreateBr(ContBB); 1162 } 1163 1164 SI->addCase(Builder.getInt32(0), BBs[0]); 1165 SI->addCase(Builder.getInt32(1), BBs[1]); 1166 SI->addCase(Builder.getInt32(2), BBs[1]); 1167 SI->addCase(Builder.getInt32(3), BBs[2]); 1168 SI->addCase(Builder.getInt32(4), BBs[3]); 1169 SI->addCase(Builder.getInt32(5), BBs[4]); 1170 1171 Builder.SetInsertPoint(ContBB); 1172 return RValue::get(Builder.CreateIsNotNull(Result, "tobool")); 1173 } 1174 1175 case Builtin::BI__atomic_clear: { 1176 QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType(); 1177 bool Volatile = 1178 PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified(); 1179 1180 Value *Ptr = EmitScalarExpr(E->getArg(0)); 1181 unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace(); 1182 Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace)); 1183 Value *NewVal = Builder.getInt8(0); 1184 Value *Order = EmitScalarExpr(E->getArg(1)); 1185 if (isa<llvm::ConstantInt>(Order)) { 1186 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1187 StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile); 1188 Store->setAlignment(1); 1189 switch (ord) { 1190 case 0: // memory_order_relaxed 1191 default: // invalid order 1192 Store->setOrdering(llvm::Monotonic); 1193 break; 1194 case 3: // memory_order_release 1195 Store->setOrdering(llvm::Release); 1196 break; 1197 case 5: // memory_order_seq_cst 1198 Store->setOrdering(llvm::SequentiallyConsistent); 1199 break; 1200 } 1201 return RValue::get(nullptr); 1202 } 1203 1204 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1205 1206 llvm::BasicBlock *BBs[3] = { 1207 createBasicBlock("monotonic", CurFn), 1208 createBasicBlock("release", CurFn), 1209 createBasicBlock("seqcst", CurFn) 1210 }; 1211 llvm::AtomicOrdering Orders[3] = { 1212 llvm::Monotonic, llvm::Release, llvm::SequentiallyConsistent 1213 }; 1214 1215 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1216 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]); 1217 1218 for (unsigned i = 0; i < 3; ++i) { 1219 Builder.SetInsertPoint(BBs[i]); 1220 StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile); 1221 Store->setAlignment(1); 1222 Store->setOrdering(Orders[i]); 1223 Builder.CreateBr(ContBB); 1224 } 1225 1226 SI->addCase(Builder.getInt32(0), BBs[0]); 1227 SI->addCase(Builder.getInt32(3), BBs[1]); 1228 SI->addCase(Builder.getInt32(5), BBs[2]); 1229 1230 Builder.SetInsertPoint(ContBB); 1231 return RValue::get(nullptr); 1232 } 1233 1234 case Builtin::BI__atomic_thread_fence: 1235 case Builtin::BI__atomic_signal_fence: 1236 case Builtin::BI__c11_atomic_thread_fence: 1237 case Builtin::BI__c11_atomic_signal_fence: { 1238 llvm::SynchronizationScope Scope; 1239 if (BuiltinID == Builtin::BI__atomic_signal_fence || 1240 BuiltinID == Builtin::BI__c11_atomic_signal_fence) 1241 Scope = llvm::SingleThread; 1242 else 1243 Scope = llvm::CrossThread; 1244 Value *Order = EmitScalarExpr(E->getArg(0)); 1245 if (isa<llvm::ConstantInt>(Order)) { 1246 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1247 switch (ord) { 1248 case 0: // memory_order_relaxed 1249 default: // invalid order 1250 break; 1251 case 1: // memory_order_consume 1252 case 2: // memory_order_acquire 1253 Builder.CreateFence(llvm::Acquire, Scope); 1254 break; 1255 case 3: // memory_order_release 1256 Builder.CreateFence(llvm::Release, Scope); 1257 break; 1258 case 4: // memory_order_acq_rel 1259 Builder.CreateFence(llvm::AcquireRelease, Scope); 1260 break; 1261 case 5: // memory_order_seq_cst 1262 Builder.CreateFence(llvm::SequentiallyConsistent, Scope); 1263 break; 1264 } 1265 return RValue::get(nullptr); 1266 } 1267 1268 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB; 1269 AcquireBB = createBasicBlock("acquire", CurFn); 1270 ReleaseBB = createBasicBlock("release", CurFn); 1271 AcqRelBB = createBasicBlock("acqrel", CurFn); 1272 SeqCstBB = createBasicBlock("seqcst", CurFn); 1273 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1274 1275 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1276 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB); 1277 1278 Builder.SetInsertPoint(AcquireBB); 1279 Builder.CreateFence(llvm::Acquire, Scope); 1280 Builder.CreateBr(ContBB); 1281 SI->addCase(Builder.getInt32(1), AcquireBB); 1282 SI->addCase(Builder.getInt32(2), AcquireBB); 1283 1284 Builder.SetInsertPoint(ReleaseBB); 1285 Builder.CreateFence(llvm::Release, Scope); 1286 Builder.CreateBr(ContBB); 1287 SI->addCase(Builder.getInt32(3), ReleaseBB); 1288 1289 Builder.SetInsertPoint(AcqRelBB); 1290 Builder.CreateFence(llvm::AcquireRelease, Scope); 1291 Builder.CreateBr(ContBB); 1292 SI->addCase(Builder.getInt32(4), AcqRelBB); 1293 1294 Builder.SetInsertPoint(SeqCstBB); 1295 Builder.CreateFence(llvm::SequentiallyConsistent, Scope); 1296 Builder.CreateBr(ContBB); 1297 SI->addCase(Builder.getInt32(5), SeqCstBB); 1298 1299 Builder.SetInsertPoint(ContBB); 1300 return RValue::get(nullptr); 1301 } 1302 1303 // Library functions with special handling. 1304 case Builtin::BIsqrt: 1305 case Builtin::BIsqrtf: 1306 case Builtin::BIsqrtl: { 1307 // Transform a call to sqrt* into a @llvm.sqrt.* intrinsic call, but only 1308 // in finite- or unsafe-math mode (the intrinsic has different semantics 1309 // for handling negative numbers compared to the library function, so 1310 // -fmath-errno=0 is not enough). 1311 if (!FD->hasAttr<ConstAttr>()) 1312 break; 1313 if (!(CGM.getCodeGenOpts().UnsafeFPMath || 1314 CGM.getCodeGenOpts().NoNaNsFPMath)) 1315 break; 1316 Value *Arg0 = EmitScalarExpr(E->getArg(0)); 1317 llvm::Type *ArgType = Arg0->getType(); 1318 Value *F = CGM.getIntrinsic(Intrinsic::sqrt, ArgType); 1319 return RValue::get(Builder.CreateCall(F, Arg0)); 1320 } 1321 1322 case Builtin::BIpow: 1323 case Builtin::BIpowf: 1324 case Builtin::BIpowl: { 1325 // Transform a call to pow* into a @llvm.pow.* intrinsic call. 1326 if (!FD->hasAttr<ConstAttr>()) 1327 break; 1328 Value *Base = EmitScalarExpr(E->getArg(0)); 1329 Value *Exponent = EmitScalarExpr(E->getArg(1)); 1330 llvm::Type *ArgType = Base->getType(); 1331 Value *F = CGM.getIntrinsic(Intrinsic::pow, ArgType); 1332 return RValue::get(Builder.CreateCall2(F, Base, Exponent)); 1333 } 1334 1335 case Builtin::BIfma: 1336 case Builtin::BIfmaf: 1337 case Builtin::BIfmal: 1338 case Builtin::BI__builtin_fma: 1339 case Builtin::BI__builtin_fmaf: 1340 case Builtin::BI__builtin_fmal: { 1341 // Rewrite fma to intrinsic. 1342 Value *FirstArg = EmitScalarExpr(E->getArg(0)); 1343 llvm::Type *ArgType = FirstArg->getType(); 1344 Value *F = CGM.getIntrinsic(Intrinsic::fma, ArgType); 1345 return RValue::get(Builder.CreateCall3(F, FirstArg, 1346 EmitScalarExpr(E->getArg(1)), 1347 EmitScalarExpr(E->getArg(2)))); 1348 } 1349 1350 case Builtin::BI__builtin_signbit: 1351 case Builtin::BI__builtin_signbitf: 1352 case Builtin::BI__builtin_signbitl: { 1353 LLVMContext &C = CGM.getLLVMContext(); 1354 1355 Value *Arg = EmitScalarExpr(E->getArg(0)); 1356 llvm::Type *ArgTy = Arg->getType(); 1357 int ArgWidth = ArgTy->getPrimitiveSizeInBits(); 1358 llvm::Type *ArgIntTy = llvm::IntegerType::get(C, ArgWidth); 1359 Value *BCArg = Builder.CreateBitCast(Arg, ArgIntTy); 1360 if (ArgTy->isPPC_FP128Ty()) { 1361 // The higher-order double comes first, and so we need to truncate the 1362 // pair to extract the overall sign. The order of the pair is the same 1363 // in both little- and big-Endian modes. 1364 ArgWidth >>= 1; 1365 ArgIntTy = llvm::IntegerType::get(C, ArgWidth); 1366 BCArg = Builder.CreateTrunc(BCArg, ArgIntTy); 1367 } 1368 Value *ZeroCmp = llvm::Constant::getNullValue(ArgIntTy); 1369 Value *Result = Builder.CreateICmpSLT(BCArg, ZeroCmp); 1370 return RValue::get(Builder.CreateZExt(Result, ConvertType(E->getType()))); 1371 } 1372 case Builtin::BI__builtin_annotation: { 1373 llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0)); 1374 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::annotation, 1375 AnnVal->getType()); 1376 1377 // Get the annotation string, go through casts. Sema requires this to be a 1378 // non-wide string literal, potentially casted, so the cast<> is safe. 1379 const Expr *AnnotationStrExpr = E->getArg(1)->IgnoreParenCasts(); 1380 StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString(); 1381 return RValue::get(EmitAnnotationCall(F, AnnVal, Str, E->getExprLoc())); 1382 } 1383 case Builtin::BI__builtin_addcb: 1384 case Builtin::BI__builtin_addcs: 1385 case Builtin::BI__builtin_addc: 1386 case Builtin::BI__builtin_addcl: 1387 case Builtin::BI__builtin_addcll: 1388 case Builtin::BI__builtin_subcb: 1389 case Builtin::BI__builtin_subcs: 1390 case Builtin::BI__builtin_subc: 1391 case Builtin::BI__builtin_subcl: 1392 case Builtin::BI__builtin_subcll: { 1393 1394 // We translate all of these builtins from expressions of the form: 1395 // int x = ..., y = ..., carryin = ..., carryout, result; 1396 // result = __builtin_addc(x, y, carryin, &carryout); 1397 // 1398 // to LLVM IR of the form: 1399 // 1400 // %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y) 1401 // %tmpsum1 = extractvalue {i32, i1} %tmp1, 0 1402 // %carry1 = extractvalue {i32, i1} %tmp1, 1 1403 // %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1, 1404 // i32 %carryin) 1405 // %result = extractvalue {i32, i1} %tmp2, 0 1406 // %carry2 = extractvalue {i32, i1} %tmp2, 1 1407 // %tmp3 = or i1 %carry1, %carry2 1408 // %tmp4 = zext i1 %tmp3 to i32 1409 // store i32 %tmp4, i32* %carryout 1410 1411 // Scalarize our inputs. 1412 llvm::Value *X = EmitScalarExpr(E->getArg(0)); 1413 llvm::Value *Y = EmitScalarExpr(E->getArg(1)); 1414 llvm::Value *Carryin = EmitScalarExpr(E->getArg(2)); 1415 std::pair<llvm::Value*, unsigned> CarryOutPtr = 1416 EmitPointerWithAlignment(E->getArg(3)); 1417 1418 // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow. 1419 llvm::Intrinsic::ID IntrinsicId; 1420 switch (BuiltinID) { 1421 default: llvm_unreachable("Unknown multiprecision builtin id."); 1422 case Builtin::BI__builtin_addcb: 1423 case Builtin::BI__builtin_addcs: 1424 case Builtin::BI__builtin_addc: 1425 case Builtin::BI__builtin_addcl: 1426 case Builtin::BI__builtin_addcll: 1427 IntrinsicId = llvm::Intrinsic::uadd_with_overflow; 1428 break; 1429 case Builtin::BI__builtin_subcb: 1430 case Builtin::BI__builtin_subcs: 1431 case Builtin::BI__builtin_subc: 1432 case Builtin::BI__builtin_subcl: 1433 case Builtin::BI__builtin_subcll: 1434 IntrinsicId = llvm::Intrinsic::usub_with_overflow; 1435 break; 1436 } 1437 1438 // Construct our resulting LLVM IR expression. 1439 llvm::Value *Carry1; 1440 llvm::Value *Sum1 = EmitOverflowIntrinsic(*this, IntrinsicId, 1441 X, Y, Carry1); 1442 llvm::Value *Carry2; 1443 llvm::Value *Sum2 = EmitOverflowIntrinsic(*this, IntrinsicId, 1444 Sum1, Carryin, Carry2); 1445 llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2), 1446 X->getType()); 1447 llvm::StoreInst *CarryOutStore = Builder.CreateStore(CarryOut, 1448 CarryOutPtr.first); 1449 CarryOutStore->setAlignment(CarryOutPtr.second); 1450 return RValue::get(Sum2); 1451 } 1452 case Builtin::BI__builtin_uadd_overflow: 1453 case Builtin::BI__builtin_uaddl_overflow: 1454 case Builtin::BI__builtin_uaddll_overflow: 1455 case Builtin::BI__builtin_usub_overflow: 1456 case Builtin::BI__builtin_usubl_overflow: 1457 case Builtin::BI__builtin_usubll_overflow: 1458 case Builtin::BI__builtin_umul_overflow: 1459 case Builtin::BI__builtin_umull_overflow: 1460 case Builtin::BI__builtin_umulll_overflow: 1461 case Builtin::BI__builtin_sadd_overflow: 1462 case Builtin::BI__builtin_saddl_overflow: 1463 case Builtin::BI__builtin_saddll_overflow: 1464 case Builtin::BI__builtin_ssub_overflow: 1465 case Builtin::BI__builtin_ssubl_overflow: 1466 case Builtin::BI__builtin_ssubll_overflow: 1467 case Builtin::BI__builtin_smul_overflow: 1468 case Builtin::BI__builtin_smull_overflow: 1469 case Builtin::BI__builtin_smulll_overflow: { 1470 1471 // We translate all of these builtins directly to the relevant llvm IR node. 1472 1473 // Scalarize our inputs. 1474 llvm::Value *X = EmitScalarExpr(E->getArg(0)); 1475 llvm::Value *Y = EmitScalarExpr(E->getArg(1)); 1476 std::pair<llvm::Value *, unsigned> SumOutPtr = 1477 EmitPointerWithAlignment(E->getArg(2)); 1478 1479 // Decide which of the overflow intrinsics we are lowering to: 1480 llvm::Intrinsic::ID IntrinsicId; 1481 switch (BuiltinID) { 1482 default: llvm_unreachable("Unknown security overflow builtin id."); 1483 case Builtin::BI__builtin_uadd_overflow: 1484 case Builtin::BI__builtin_uaddl_overflow: 1485 case Builtin::BI__builtin_uaddll_overflow: 1486 IntrinsicId = llvm::Intrinsic::uadd_with_overflow; 1487 break; 1488 case Builtin::BI__builtin_usub_overflow: 1489 case Builtin::BI__builtin_usubl_overflow: 1490 case Builtin::BI__builtin_usubll_overflow: 1491 IntrinsicId = llvm::Intrinsic::usub_with_overflow; 1492 break; 1493 case Builtin::BI__builtin_umul_overflow: 1494 case Builtin::BI__builtin_umull_overflow: 1495 case Builtin::BI__builtin_umulll_overflow: 1496 IntrinsicId = llvm::Intrinsic::umul_with_overflow; 1497 break; 1498 case Builtin::BI__builtin_sadd_overflow: 1499 case Builtin::BI__builtin_saddl_overflow: 1500 case Builtin::BI__builtin_saddll_overflow: 1501 IntrinsicId = llvm::Intrinsic::sadd_with_overflow; 1502 break; 1503 case Builtin::BI__builtin_ssub_overflow: 1504 case Builtin::BI__builtin_ssubl_overflow: 1505 case Builtin::BI__builtin_ssubll_overflow: 1506 IntrinsicId = llvm::Intrinsic::ssub_with_overflow; 1507 break; 1508 case Builtin::BI__builtin_smul_overflow: 1509 case Builtin::BI__builtin_smull_overflow: 1510 case Builtin::BI__builtin_smulll_overflow: 1511 IntrinsicId = llvm::Intrinsic::smul_with_overflow; 1512 break; 1513 } 1514 1515 1516 llvm::Value *Carry; 1517 llvm::Value *Sum = EmitOverflowIntrinsic(*this, IntrinsicId, X, Y, Carry); 1518 llvm::StoreInst *SumOutStore = Builder.CreateStore(Sum, SumOutPtr.first); 1519 SumOutStore->setAlignment(SumOutPtr.second); 1520 1521 return RValue::get(Carry); 1522 } 1523 case Builtin::BI__builtin_addressof: 1524 return RValue::get(EmitLValue(E->getArg(0)).getAddress()); 1525 case Builtin::BI__builtin_operator_new: 1526 return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(), 1527 E->getArg(0), false); 1528 case Builtin::BI__builtin_operator_delete: 1529 return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(), 1530 E->getArg(0), true); 1531 case Builtin::BI__noop: 1532 // __noop always evaluates to an integer literal zero. 1533 return RValue::get(ConstantInt::get(IntTy, 0)); 1534 case Builtin::BI_InterlockedExchange: 1535 case Builtin::BI_InterlockedExchangePointer: 1536 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); 1537 case Builtin::BI_InterlockedCompareExchangePointer: { 1538 llvm::Type *RTy; 1539 llvm::IntegerType *IntType = 1540 IntegerType::get(getLLVMContext(), 1541 getContext().getTypeSize(E->getType())); 1542 llvm::Type *IntPtrType = IntType->getPointerTo(); 1543 1544 llvm::Value *Destination = 1545 Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), IntPtrType); 1546 1547 llvm::Value *Exchange = EmitScalarExpr(E->getArg(1)); 1548 RTy = Exchange->getType(); 1549 Exchange = Builder.CreatePtrToInt(Exchange, IntType); 1550 1551 llvm::Value *Comparand = 1552 Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType); 1553 1554 auto Result = Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange, 1555 SequentiallyConsistent, 1556 SequentiallyConsistent); 1557 Result->setVolatile(true); 1558 1559 return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result, 1560 0), 1561 RTy)); 1562 } 1563 case Builtin::BI_InterlockedCompareExchange: { 1564 AtomicCmpXchgInst *CXI = Builder.CreateAtomicCmpXchg( 1565 EmitScalarExpr(E->getArg(0)), 1566 EmitScalarExpr(E->getArg(2)), 1567 EmitScalarExpr(E->getArg(1)), 1568 SequentiallyConsistent, 1569 SequentiallyConsistent); 1570 CXI->setVolatile(true); 1571 return RValue::get(Builder.CreateExtractValue(CXI, 0)); 1572 } 1573 case Builtin::BI_InterlockedIncrement: { 1574 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1575 AtomicRMWInst::Add, 1576 EmitScalarExpr(E->getArg(0)), 1577 ConstantInt::get(Int32Ty, 1), 1578 llvm::SequentiallyConsistent); 1579 RMWI->setVolatile(true); 1580 return RValue::get(Builder.CreateAdd(RMWI, ConstantInt::get(Int32Ty, 1))); 1581 } 1582 case Builtin::BI_InterlockedDecrement: { 1583 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1584 AtomicRMWInst::Sub, 1585 EmitScalarExpr(E->getArg(0)), 1586 ConstantInt::get(Int32Ty, 1), 1587 llvm::SequentiallyConsistent); 1588 RMWI->setVolatile(true); 1589 return RValue::get(Builder.CreateSub(RMWI, ConstantInt::get(Int32Ty, 1))); 1590 } 1591 case Builtin::BI_InterlockedExchangeAdd: { 1592 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1593 AtomicRMWInst::Add, 1594 EmitScalarExpr(E->getArg(0)), 1595 EmitScalarExpr(E->getArg(1)), 1596 llvm::SequentiallyConsistent); 1597 RMWI->setVolatile(true); 1598 return RValue::get(RMWI); 1599 } 1600 } 1601 1602 // If this is an alias for a lib function (e.g. __builtin_sin), emit 1603 // the call using the normal call path, but using the unmangled 1604 // version of the function name. 1605 if (getContext().BuiltinInfo.isLibFunction(BuiltinID)) 1606 return emitLibraryCall(*this, FD, E, 1607 CGM.getBuiltinLibFunction(FD, BuiltinID)); 1608 1609 // If this is a predefined lib function (e.g. malloc), emit the call 1610 // using exactly the normal call path. 1611 if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID)) 1612 return emitLibraryCall(*this, FD, E, EmitScalarExpr(E->getCallee())); 1613 1614 // See if we have a target specific intrinsic. 1615 const char *Name = getContext().BuiltinInfo.GetName(BuiltinID); 1616 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic; 1617 if (const char *Prefix = 1618 llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch())) { 1619 IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix, Name); 1620 // NOTE we dont need to perform a compatibility flag check here since the 1621 // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the 1622 // MS builtins via ALL_MS_LANGUAGES and are filtered earlier. 1623 if (IntrinsicID == Intrinsic::not_intrinsic) 1624 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix, Name); 1625 } 1626 1627 if (IntrinsicID != Intrinsic::not_intrinsic) { 1628 SmallVector<Value*, 16> Args; 1629 1630 // Find out if any arguments are required to be integer constant 1631 // expressions. 1632 unsigned ICEArguments = 0; 1633 ASTContext::GetBuiltinTypeError Error; 1634 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 1635 assert(Error == ASTContext::GE_None && "Should not codegen an error"); 1636 1637 Function *F = CGM.getIntrinsic(IntrinsicID); 1638 llvm::FunctionType *FTy = F->getFunctionType(); 1639 1640 for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) { 1641 Value *ArgValue; 1642 // If this is a normal argument, just emit it as a scalar. 1643 if ((ICEArguments & (1 << i)) == 0) { 1644 ArgValue = EmitScalarExpr(E->getArg(i)); 1645 } else { 1646 // If this is required to be a constant, constant fold it so that we 1647 // know that the generated intrinsic gets a ConstantInt. 1648 llvm::APSInt Result; 1649 bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result,getContext()); 1650 assert(IsConst && "Constant arg isn't actually constant?"); 1651 (void)IsConst; 1652 ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result); 1653 } 1654 1655 // If the intrinsic arg type is different from the builtin arg type 1656 // we need to do a bit cast. 1657 llvm::Type *PTy = FTy->getParamType(i); 1658 if (PTy != ArgValue->getType()) { 1659 assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) && 1660 "Must be able to losslessly bit cast to param"); 1661 ArgValue = Builder.CreateBitCast(ArgValue, PTy); 1662 } 1663 1664 Args.push_back(ArgValue); 1665 } 1666 1667 Value *V = Builder.CreateCall(F, Args); 1668 QualType BuiltinRetType = E->getType(); 1669 1670 llvm::Type *RetTy = VoidTy; 1671 if (!BuiltinRetType->isVoidType()) 1672 RetTy = ConvertType(BuiltinRetType); 1673 1674 if (RetTy != V->getType()) { 1675 assert(V->getType()->canLosslesslyBitCastTo(RetTy) && 1676 "Must be able to losslessly bit cast result type"); 1677 V = Builder.CreateBitCast(V, RetTy); 1678 } 1679 1680 return RValue::get(V); 1681 } 1682 1683 // See if we have a target specific builtin that needs to be lowered. 1684 if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E)) 1685 return RValue::get(V); 1686 1687 ErrorUnsupported(E, "builtin function"); 1688 1689 // Unknown builtin, for now just dump it out and return undef. 1690 return GetUndefRValue(E->getType()); 1691 } 1692 1693 Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID, 1694 const CallExpr *E) { 1695 switch (getTarget().getTriple().getArch()) { 1696 case llvm::Triple::arm: 1697 case llvm::Triple::armeb: 1698 case llvm::Triple::thumb: 1699 case llvm::Triple::thumbeb: 1700 return EmitARMBuiltinExpr(BuiltinID, E); 1701 case llvm::Triple::aarch64: 1702 case llvm::Triple::aarch64_be: 1703 return EmitAArch64BuiltinExpr(BuiltinID, E); 1704 case llvm::Triple::x86: 1705 case llvm::Triple::x86_64: 1706 return EmitX86BuiltinExpr(BuiltinID, E); 1707 case llvm::Triple::ppc: 1708 case llvm::Triple::ppc64: 1709 case llvm::Triple::ppc64le: 1710 return EmitPPCBuiltinExpr(BuiltinID, E); 1711 case llvm::Triple::r600: 1712 return EmitR600BuiltinExpr(BuiltinID, E); 1713 default: 1714 return nullptr; 1715 } 1716 } 1717 1718 static llvm::VectorType *GetNeonType(CodeGenFunction *CGF, 1719 NeonTypeFlags TypeFlags, 1720 bool V1Ty=false) { 1721 int IsQuad = TypeFlags.isQuad(); 1722 switch (TypeFlags.getEltType()) { 1723 case NeonTypeFlags::Int8: 1724 case NeonTypeFlags::Poly8: 1725 return llvm::VectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad)); 1726 case NeonTypeFlags::Int16: 1727 case NeonTypeFlags::Poly16: 1728 case NeonTypeFlags::Float16: 1729 return llvm::VectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad)); 1730 case NeonTypeFlags::Int32: 1731 return llvm::VectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad)); 1732 case NeonTypeFlags::Int64: 1733 case NeonTypeFlags::Poly64: 1734 return llvm::VectorType::get(CGF->Int64Ty, V1Ty ? 1 : (1 << IsQuad)); 1735 case NeonTypeFlags::Poly128: 1736 // FIXME: i128 and f128 doesn't get fully support in Clang and llvm. 1737 // There is a lot of i128 and f128 API missing. 1738 // so we use v16i8 to represent poly128 and get pattern matched. 1739 return llvm::VectorType::get(CGF->Int8Ty, 16); 1740 case NeonTypeFlags::Float32: 1741 return llvm::VectorType::get(CGF->FloatTy, V1Ty ? 1 : (2 << IsQuad)); 1742 case NeonTypeFlags::Float64: 1743 return llvm::VectorType::get(CGF->DoubleTy, V1Ty ? 1 : (1 << IsQuad)); 1744 } 1745 llvm_unreachable("Unknown vector element type!"); 1746 } 1747 1748 Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) { 1749 unsigned nElts = cast<llvm::VectorType>(V->getType())->getNumElements(); 1750 Value* SV = llvm::ConstantVector::getSplat(nElts, C); 1751 return Builder.CreateShuffleVector(V, V, SV, "lane"); 1752 } 1753 1754 Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops, 1755 const char *name, 1756 unsigned shift, bool rightshift) { 1757 unsigned j = 0; 1758 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end(); 1759 ai != ae; ++ai, ++j) 1760 if (shift > 0 && shift == j) 1761 Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift); 1762 else 1763 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name); 1764 1765 return Builder.CreateCall(F, Ops, name); 1766 } 1767 1768 Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty, 1769 bool neg) { 1770 int SV = cast<ConstantInt>(V)->getSExtValue(); 1771 1772 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); 1773 llvm::Constant *C = ConstantInt::get(VTy->getElementType(), neg ? -SV : SV); 1774 return llvm::ConstantVector::getSplat(VTy->getNumElements(), C); 1775 } 1776 1777 // \brief Right-shift a vector by a constant. 1778 Value *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift, 1779 llvm::Type *Ty, bool usgn, 1780 const char *name) { 1781 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); 1782 1783 int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue(); 1784 int EltSize = VTy->getScalarSizeInBits(); 1785 1786 Vec = Builder.CreateBitCast(Vec, Ty); 1787 1788 // lshr/ashr are undefined when the shift amount is equal to the vector 1789 // element size. 1790 if (ShiftAmt == EltSize) { 1791 if (usgn) { 1792 // Right-shifting an unsigned value by its size yields 0. 1793 llvm::Constant *Zero = ConstantInt::get(VTy->getElementType(), 0); 1794 return llvm::ConstantVector::getSplat(VTy->getNumElements(), Zero); 1795 } else { 1796 // Right-shifting a signed value by its size is equivalent 1797 // to a shift of size-1. 1798 --ShiftAmt; 1799 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt); 1800 } 1801 } 1802 1803 Shift = EmitNeonShiftVector(Shift, Ty, false); 1804 if (usgn) 1805 return Builder.CreateLShr(Vec, Shift, name); 1806 else 1807 return Builder.CreateAShr(Vec, Shift, name); 1808 } 1809 1810 /// GetPointeeAlignment - Given an expression with a pointer type, find the 1811 /// alignment of the type referenced by the pointer. Skip over implicit 1812 /// casts. 1813 std::pair<llvm::Value*, unsigned> 1814 CodeGenFunction::EmitPointerWithAlignment(const Expr *Addr) { 1815 assert(Addr->getType()->isPointerType()); 1816 Addr = Addr->IgnoreParens(); 1817 if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(Addr)) { 1818 if ((ICE->getCastKind() == CK_BitCast || ICE->getCastKind() == CK_NoOp) && 1819 ICE->getSubExpr()->getType()->isPointerType()) { 1820 std::pair<llvm::Value*, unsigned> Ptr = 1821 EmitPointerWithAlignment(ICE->getSubExpr()); 1822 Ptr.first = Builder.CreateBitCast(Ptr.first, 1823 ConvertType(Addr->getType())); 1824 return Ptr; 1825 } else if (ICE->getCastKind() == CK_ArrayToPointerDecay) { 1826 LValue LV = EmitLValue(ICE->getSubExpr()); 1827 unsigned Align = LV.getAlignment().getQuantity(); 1828 if (!Align) { 1829 // FIXME: Once LValues are fixed to always set alignment, 1830 // zap this code. 1831 QualType PtTy = ICE->getSubExpr()->getType(); 1832 if (!PtTy->isIncompleteType()) 1833 Align = getContext().getTypeAlignInChars(PtTy).getQuantity(); 1834 else 1835 Align = 1; 1836 } 1837 return std::make_pair(LV.getAddress(), Align); 1838 } 1839 } 1840 if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(Addr)) { 1841 if (UO->getOpcode() == UO_AddrOf) { 1842 LValue LV = EmitLValue(UO->getSubExpr()); 1843 unsigned Align = LV.getAlignment().getQuantity(); 1844 if (!Align) { 1845 // FIXME: Once LValues are fixed to always set alignment, 1846 // zap this code. 1847 QualType PtTy = UO->getSubExpr()->getType(); 1848 if (!PtTy->isIncompleteType()) 1849 Align = getContext().getTypeAlignInChars(PtTy).getQuantity(); 1850 else 1851 Align = 1; 1852 } 1853 return std::make_pair(LV.getAddress(), Align); 1854 } 1855 } 1856 1857 unsigned Align = 1; 1858 QualType PtTy = Addr->getType()->getPointeeType(); 1859 if (!PtTy->isIncompleteType()) 1860 Align = getContext().getTypeAlignInChars(PtTy).getQuantity(); 1861 1862 return std::make_pair(EmitScalarExpr(Addr), Align); 1863 } 1864 1865 enum { 1866 AddRetType = (1 << 0), 1867 Add1ArgType = (1 << 1), 1868 Add2ArgTypes = (1 << 2), 1869 1870 VectorizeRetType = (1 << 3), 1871 VectorizeArgTypes = (1 << 4), 1872 1873 InventFloatType = (1 << 5), 1874 UnsignedAlts = (1 << 6), 1875 1876 Use64BitVectors = (1 << 7), 1877 Use128BitVectors = (1 << 8), 1878 1879 Vectorize1ArgType = Add1ArgType | VectorizeArgTypes, 1880 VectorRet = AddRetType | VectorizeRetType, 1881 VectorRetGetArgs01 = 1882 AddRetType | Add2ArgTypes | VectorizeRetType | VectorizeArgTypes, 1883 FpCmpzModifiers = 1884 AddRetType | VectorizeRetType | Add1ArgType | InventFloatType 1885 }; 1886 1887 struct NeonIntrinsicInfo { 1888 unsigned BuiltinID; 1889 unsigned LLVMIntrinsic; 1890 unsigned AltLLVMIntrinsic; 1891 const char *NameHint; 1892 unsigned TypeModifier; 1893 1894 bool operator<(unsigned RHSBuiltinID) const { 1895 return BuiltinID < RHSBuiltinID; 1896 } 1897 }; 1898 1899 #define NEONMAP0(NameBase) \ 1900 { NEON::BI__builtin_neon_ ## NameBase, 0, 0, #NameBase, 0 } 1901 1902 #define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \ 1903 { NEON:: BI__builtin_neon_ ## NameBase, \ 1904 Intrinsic::LLVMIntrinsic, 0, #NameBase, TypeModifier } 1905 1906 #define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \ 1907 { NEON:: BI__builtin_neon_ ## NameBase, \ 1908 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \ 1909 #NameBase, TypeModifier } 1910 1911 static NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = { 1912 NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts), 1913 NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts), 1914 NEONMAP1(vabs_v, arm_neon_vabs, 0), 1915 NEONMAP1(vabsq_v, arm_neon_vabs, 0), 1916 NEONMAP0(vaddhn_v), 1917 NEONMAP1(vaesdq_v, arm_neon_aesd, 0), 1918 NEONMAP1(vaeseq_v, arm_neon_aese, 0), 1919 NEONMAP1(vaesimcq_v, arm_neon_aesimc, 0), 1920 NEONMAP1(vaesmcq_v, arm_neon_aesmc, 0), 1921 NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType), 1922 NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType), 1923 NEONMAP1(vcage_v, arm_neon_vacge, 0), 1924 NEONMAP1(vcageq_v, arm_neon_vacge, 0), 1925 NEONMAP1(vcagt_v, arm_neon_vacgt, 0), 1926 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0), 1927 NEONMAP1(vcale_v, arm_neon_vacge, 0), 1928 NEONMAP1(vcaleq_v, arm_neon_vacge, 0), 1929 NEONMAP1(vcalt_v, arm_neon_vacgt, 0), 1930 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0), 1931 NEONMAP1(vcls_v, arm_neon_vcls, Add1ArgType), 1932 NEONMAP1(vclsq_v, arm_neon_vcls, Add1ArgType), 1933 NEONMAP1(vclz_v, ctlz, Add1ArgType), 1934 NEONMAP1(vclzq_v, ctlz, Add1ArgType), 1935 NEONMAP1(vcnt_v, ctpop, Add1ArgType), 1936 NEONMAP1(vcntq_v, ctpop, Add1ArgType), 1937 NEONMAP1(vcvt_f16_v, arm_neon_vcvtfp2hf, 0), 1938 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0), 1939 NEONMAP0(vcvt_f32_v), 1940 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), 1941 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0), 1942 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0), 1943 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0), 1944 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0), 1945 NEONMAP0(vcvt_s32_v), 1946 NEONMAP0(vcvt_s64_v), 1947 NEONMAP0(vcvt_u32_v), 1948 NEONMAP0(vcvt_u64_v), 1949 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0), 1950 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0), 1951 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0), 1952 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0), 1953 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0), 1954 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0), 1955 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0), 1956 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0), 1957 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0), 1958 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0), 1959 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0), 1960 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0), 1961 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0), 1962 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0), 1963 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0), 1964 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0), 1965 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0), 1966 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0), 1967 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0), 1968 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0), 1969 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0), 1970 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0), 1971 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0), 1972 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0), 1973 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0), 1974 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0), 1975 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0), 1976 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0), 1977 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0), 1978 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0), 1979 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0), 1980 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0), 1981 NEONMAP0(vcvtq_f32_v), 1982 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), 1983 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0), 1984 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0), 1985 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0), 1986 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0), 1987 NEONMAP0(vcvtq_s32_v), 1988 NEONMAP0(vcvtq_s64_v), 1989 NEONMAP0(vcvtq_u32_v), 1990 NEONMAP0(vcvtq_u64_v), 1991 NEONMAP0(vext_v), 1992 NEONMAP0(vextq_v), 1993 NEONMAP0(vfma_v), 1994 NEONMAP0(vfmaq_v), 1995 NEONMAP2(vhadd_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts), 1996 NEONMAP2(vhaddq_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts), 1997 NEONMAP2(vhsub_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts), 1998 NEONMAP2(vhsubq_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts), 1999 NEONMAP0(vld1_dup_v), 2000 NEONMAP1(vld1_v, arm_neon_vld1, 0), 2001 NEONMAP0(vld1q_dup_v), 2002 NEONMAP1(vld1q_v, arm_neon_vld1, 0), 2003 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0), 2004 NEONMAP1(vld2_v, arm_neon_vld2, 0), 2005 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0), 2006 NEONMAP1(vld2q_v, arm_neon_vld2, 0), 2007 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0), 2008 NEONMAP1(vld3_v, arm_neon_vld3, 0), 2009 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0), 2010 NEONMAP1(vld3q_v, arm_neon_vld3, 0), 2011 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0), 2012 NEONMAP1(vld4_v, arm_neon_vld4, 0), 2013 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0), 2014 NEONMAP1(vld4q_v, arm_neon_vld4, 0), 2015 NEONMAP2(vmax_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts), 2016 NEONMAP1(vmaxnm_v, arm_neon_vmaxnm, Add1ArgType), 2017 NEONMAP1(vmaxnmq_v, arm_neon_vmaxnm, Add1ArgType), 2018 NEONMAP2(vmaxq_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts), 2019 NEONMAP2(vmin_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts), 2020 NEONMAP1(vminnm_v, arm_neon_vminnm, Add1ArgType), 2021 NEONMAP1(vminnmq_v, arm_neon_vminnm, Add1ArgType), 2022 NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts), 2023 NEONMAP0(vmovl_v), 2024 NEONMAP0(vmovn_v), 2025 NEONMAP1(vmul_v, arm_neon_vmulp, Add1ArgType), 2026 NEONMAP0(vmull_v), 2027 NEONMAP1(vmulq_v, arm_neon_vmulp, Add1ArgType), 2028 NEONMAP2(vpadal_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts), 2029 NEONMAP2(vpadalq_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts), 2030 NEONMAP1(vpadd_v, arm_neon_vpadd, Add1ArgType), 2031 NEONMAP2(vpaddl_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts), 2032 NEONMAP2(vpaddlq_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts), 2033 NEONMAP1(vpaddq_v, arm_neon_vpadd, Add1ArgType), 2034 NEONMAP2(vpmax_v, arm_neon_vpmaxu, arm_neon_vpmaxs, Add1ArgType | UnsignedAlts), 2035 NEONMAP2(vpmin_v, arm_neon_vpminu, arm_neon_vpmins, Add1ArgType | UnsignedAlts), 2036 NEONMAP1(vqabs_v, arm_neon_vqabs, Add1ArgType), 2037 NEONMAP1(vqabsq_v, arm_neon_vqabs, Add1ArgType), 2038 NEONMAP2(vqadd_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts), 2039 NEONMAP2(vqaddq_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts), 2040 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, arm_neon_vqadds, 0), 2041 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, arm_neon_vqsubs, 0), 2042 NEONMAP1(vqdmulh_v, arm_neon_vqdmulh, Add1ArgType), 2043 NEONMAP1(vqdmulhq_v, arm_neon_vqdmulh, Add1ArgType), 2044 NEONMAP1(vqdmull_v, arm_neon_vqdmull, Add1ArgType), 2045 NEONMAP2(vqmovn_v, arm_neon_vqmovnu, arm_neon_vqmovns, Add1ArgType | UnsignedAlts), 2046 NEONMAP1(vqmovun_v, arm_neon_vqmovnsu, Add1ArgType), 2047 NEONMAP1(vqneg_v, arm_neon_vqneg, Add1ArgType), 2048 NEONMAP1(vqnegq_v, arm_neon_vqneg, Add1ArgType), 2049 NEONMAP1(vqrdmulh_v, arm_neon_vqrdmulh, Add1ArgType), 2050 NEONMAP1(vqrdmulhq_v, arm_neon_vqrdmulh, Add1ArgType), 2051 NEONMAP2(vqrshl_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts), 2052 NEONMAP2(vqrshlq_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts), 2053 NEONMAP2(vqshl_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts), 2054 NEONMAP2(vqshl_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts), 2055 NEONMAP2(vqshlq_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts), 2056 NEONMAP2(vqshlq_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts), 2057 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0), 2058 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0), 2059 NEONMAP2(vqsub_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts), 2060 NEONMAP2(vqsubq_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts), 2061 NEONMAP1(vraddhn_v, arm_neon_vraddhn, Add1ArgType), 2062 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0), 2063 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0), 2064 NEONMAP1(vrecps_v, arm_neon_vrecps, Add1ArgType), 2065 NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType), 2066 NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts), 2067 NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts), 2068 NEONMAP1(vrnd_v, arm_neon_vrintz, Add1ArgType), 2069 NEONMAP1(vrnda_v, arm_neon_vrinta, Add1ArgType), 2070 NEONMAP1(vrndaq_v, arm_neon_vrinta, Add1ArgType), 2071 NEONMAP1(vrndm_v, arm_neon_vrintm, Add1ArgType), 2072 NEONMAP1(vrndmq_v, arm_neon_vrintm, Add1ArgType), 2073 NEONMAP1(vrndn_v, arm_neon_vrintn, Add1ArgType), 2074 NEONMAP1(vrndnq_v, arm_neon_vrintn, Add1ArgType), 2075 NEONMAP1(vrndp_v, arm_neon_vrintp, Add1ArgType), 2076 NEONMAP1(vrndpq_v, arm_neon_vrintp, Add1ArgType), 2077 NEONMAP1(vrndq_v, arm_neon_vrintz, Add1ArgType), 2078 NEONMAP1(vrndx_v, arm_neon_vrintx, Add1ArgType), 2079 NEONMAP1(vrndxq_v, arm_neon_vrintx, Add1ArgType), 2080 NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts), 2081 NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts), 2082 NEONMAP2(vrshr_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts), 2083 NEONMAP2(vrshrq_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts), 2084 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0), 2085 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0), 2086 NEONMAP1(vrsqrts_v, arm_neon_vrsqrts, Add1ArgType), 2087 NEONMAP1(vrsqrtsq_v, arm_neon_vrsqrts, Add1ArgType), 2088 NEONMAP1(vrsubhn_v, arm_neon_vrsubhn, Add1ArgType), 2089 NEONMAP1(vsha1su0q_v, arm_neon_sha1su0, 0), 2090 NEONMAP1(vsha1su1q_v, arm_neon_sha1su1, 0), 2091 NEONMAP1(vsha256h2q_v, arm_neon_sha256h2, 0), 2092 NEONMAP1(vsha256hq_v, arm_neon_sha256h, 0), 2093 NEONMAP1(vsha256su0q_v, arm_neon_sha256su0, 0), 2094 NEONMAP1(vsha256su1q_v, arm_neon_sha256su1, 0), 2095 NEONMAP0(vshl_n_v), 2096 NEONMAP2(vshl_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts), 2097 NEONMAP0(vshll_n_v), 2098 NEONMAP0(vshlq_n_v), 2099 NEONMAP2(vshlq_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts), 2100 NEONMAP0(vshr_n_v), 2101 NEONMAP0(vshrn_n_v), 2102 NEONMAP0(vshrq_n_v), 2103 NEONMAP1(vst1_v, arm_neon_vst1, 0), 2104 NEONMAP1(vst1q_v, arm_neon_vst1, 0), 2105 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0), 2106 NEONMAP1(vst2_v, arm_neon_vst2, 0), 2107 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0), 2108 NEONMAP1(vst2q_v, arm_neon_vst2, 0), 2109 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0), 2110 NEONMAP1(vst3_v, arm_neon_vst3, 0), 2111 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0), 2112 NEONMAP1(vst3q_v, arm_neon_vst3, 0), 2113 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0), 2114 NEONMAP1(vst4_v, arm_neon_vst4, 0), 2115 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0), 2116 NEONMAP1(vst4q_v, arm_neon_vst4, 0), 2117 NEONMAP0(vsubhn_v), 2118 NEONMAP0(vtrn_v), 2119 NEONMAP0(vtrnq_v), 2120 NEONMAP0(vtst_v), 2121 NEONMAP0(vtstq_v), 2122 NEONMAP0(vuzp_v), 2123 NEONMAP0(vuzpq_v), 2124 NEONMAP0(vzip_v), 2125 NEONMAP0(vzipq_v) 2126 }; 2127 2128 static NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = { 2129 NEONMAP1(vabs_v, aarch64_neon_abs, 0), 2130 NEONMAP1(vabsq_v, aarch64_neon_abs, 0), 2131 NEONMAP0(vaddhn_v), 2132 NEONMAP1(vaesdq_v, aarch64_crypto_aesd, 0), 2133 NEONMAP1(vaeseq_v, aarch64_crypto_aese, 0), 2134 NEONMAP1(vaesimcq_v, aarch64_crypto_aesimc, 0), 2135 NEONMAP1(vaesmcq_v, aarch64_crypto_aesmc, 0), 2136 NEONMAP1(vcage_v, aarch64_neon_facge, 0), 2137 NEONMAP1(vcageq_v, aarch64_neon_facge, 0), 2138 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0), 2139 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0), 2140 NEONMAP1(vcale_v, aarch64_neon_facge, 0), 2141 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0), 2142 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0), 2143 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0), 2144 NEONMAP1(vcls_v, aarch64_neon_cls, Add1ArgType), 2145 NEONMAP1(vclsq_v, aarch64_neon_cls, Add1ArgType), 2146 NEONMAP1(vclz_v, ctlz, Add1ArgType), 2147 NEONMAP1(vclzq_v, ctlz, Add1ArgType), 2148 NEONMAP1(vcnt_v, ctpop, Add1ArgType), 2149 NEONMAP1(vcntq_v, ctpop, Add1ArgType), 2150 NEONMAP1(vcvt_f16_v, aarch64_neon_vcvtfp2hf, 0), 2151 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0), 2152 NEONMAP0(vcvt_f32_v), 2153 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 2154 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 2155 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0), 2156 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0), 2157 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0), 2158 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0), 2159 NEONMAP0(vcvtq_f32_v), 2160 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 2161 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 2162 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0), 2163 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0), 2164 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0), 2165 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0), 2166 NEONMAP1(vcvtx_f32_v, aarch64_neon_fcvtxn, AddRetType | Add1ArgType), 2167 NEONMAP0(vext_v), 2168 NEONMAP0(vextq_v), 2169 NEONMAP0(vfma_v), 2170 NEONMAP0(vfmaq_v), 2171 NEONMAP2(vhadd_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts), 2172 NEONMAP2(vhaddq_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts), 2173 NEONMAP2(vhsub_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts), 2174 NEONMAP2(vhsubq_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts), 2175 NEONMAP0(vmovl_v), 2176 NEONMAP0(vmovn_v), 2177 NEONMAP1(vmul_v, aarch64_neon_pmul, Add1ArgType), 2178 NEONMAP1(vmulq_v, aarch64_neon_pmul, Add1ArgType), 2179 NEONMAP1(vpadd_v, aarch64_neon_addp, Add1ArgType), 2180 NEONMAP2(vpaddl_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts), 2181 NEONMAP2(vpaddlq_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts), 2182 NEONMAP1(vpaddq_v, aarch64_neon_addp, Add1ArgType), 2183 NEONMAP1(vqabs_v, aarch64_neon_sqabs, Add1ArgType), 2184 NEONMAP1(vqabsq_v, aarch64_neon_sqabs, Add1ArgType), 2185 NEONMAP2(vqadd_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts), 2186 NEONMAP2(vqaddq_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts), 2187 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0), 2188 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0), 2189 NEONMAP1(vqdmulh_v, aarch64_neon_sqdmulh, Add1ArgType), 2190 NEONMAP1(vqdmulhq_v, aarch64_neon_sqdmulh, Add1ArgType), 2191 NEONMAP1(vqdmull_v, aarch64_neon_sqdmull, Add1ArgType), 2192 NEONMAP2(vqmovn_v, aarch64_neon_uqxtn, aarch64_neon_sqxtn, Add1ArgType | UnsignedAlts), 2193 NEONMAP1(vqmovun_v, aarch64_neon_sqxtun, Add1ArgType), 2194 NEONMAP1(vqneg_v, aarch64_neon_sqneg, Add1ArgType), 2195 NEONMAP1(vqnegq_v, aarch64_neon_sqneg, Add1ArgType), 2196 NEONMAP1(vqrdmulh_v, aarch64_neon_sqrdmulh, Add1ArgType), 2197 NEONMAP1(vqrdmulhq_v, aarch64_neon_sqrdmulh, Add1ArgType), 2198 NEONMAP2(vqrshl_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts), 2199 NEONMAP2(vqrshlq_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts), 2200 NEONMAP2(vqshl_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl, UnsignedAlts), 2201 NEONMAP2(vqshl_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts), 2202 NEONMAP2(vqshlq_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl,UnsignedAlts), 2203 NEONMAP2(vqshlq_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts), 2204 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0), 2205 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0), 2206 NEONMAP2(vqsub_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts), 2207 NEONMAP2(vqsubq_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts), 2208 NEONMAP1(vraddhn_v, aarch64_neon_raddhn, Add1ArgType), 2209 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0), 2210 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0), 2211 NEONMAP1(vrecps_v, aarch64_neon_frecps, Add1ArgType), 2212 NEONMAP1(vrecpsq_v, aarch64_neon_frecps, Add1ArgType), 2213 NEONMAP2(vrhadd_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts), 2214 NEONMAP2(vrhaddq_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts), 2215 NEONMAP2(vrshl_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts), 2216 NEONMAP2(vrshlq_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts), 2217 NEONMAP2(vrshr_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts), 2218 NEONMAP2(vrshrq_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts), 2219 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0), 2220 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0), 2221 NEONMAP1(vrsqrts_v, aarch64_neon_frsqrts, Add1ArgType), 2222 NEONMAP1(vrsqrtsq_v, aarch64_neon_frsqrts, Add1ArgType), 2223 NEONMAP1(vrsubhn_v, aarch64_neon_rsubhn, Add1ArgType), 2224 NEONMAP1(vsha1su0q_v, aarch64_crypto_sha1su0, 0), 2225 NEONMAP1(vsha1su1q_v, aarch64_crypto_sha1su1, 0), 2226 NEONMAP1(vsha256h2q_v, aarch64_crypto_sha256h2, 0), 2227 NEONMAP1(vsha256hq_v, aarch64_crypto_sha256h, 0), 2228 NEONMAP1(vsha256su0q_v, aarch64_crypto_sha256su0, 0), 2229 NEONMAP1(vsha256su1q_v, aarch64_crypto_sha256su1, 0), 2230 NEONMAP0(vshl_n_v), 2231 NEONMAP2(vshl_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts), 2232 NEONMAP0(vshll_n_v), 2233 NEONMAP0(vshlq_n_v), 2234 NEONMAP2(vshlq_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts), 2235 NEONMAP0(vshr_n_v), 2236 NEONMAP0(vshrn_n_v), 2237 NEONMAP0(vshrq_n_v), 2238 NEONMAP0(vsubhn_v), 2239 NEONMAP0(vtst_v), 2240 NEONMAP0(vtstq_v), 2241 }; 2242 2243 static NeonIntrinsicInfo AArch64SISDIntrinsicMap[] = { 2244 NEONMAP1(vabdd_f64, aarch64_sisd_fabd, Add1ArgType), 2245 NEONMAP1(vabds_f32, aarch64_sisd_fabd, Add1ArgType), 2246 NEONMAP1(vabsd_s64, aarch64_neon_abs, Add1ArgType), 2247 NEONMAP1(vaddlv_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType), 2248 NEONMAP1(vaddlv_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType), 2249 NEONMAP1(vaddlvq_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType), 2250 NEONMAP1(vaddlvq_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType), 2251 NEONMAP1(vaddv_f32, aarch64_neon_faddv, AddRetType | Add1ArgType), 2252 NEONMAP1(vaddv_s32, aarch64_neon_saddv, AddRetType | Add1ArgType), 2253 NEONMAP1(vaddv_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType), 2254 NEONMAP1(vaddvq_f32, aarch64_neon_faddv, AddRetType | Add1ArgType), 2255 NEONMAP1(vaddvq_f64, aarch64_neon_faddv, AddRetType | Add1ArgType), 2256 NEONMAP1(vaddvq_s32, aarch64_neon_saddv, AddRetType | Add1ArgType), 2257 NEONMAP1(vaddvq_s64, aarch64_neon_saddv, AddRetType | Add1ArgType), 2258 NEONMAP1(vaddvq_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType), 2259 NEONMAP1(vaddvq_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 2260 NEONMAP1(vcaged_f64, aarch64_neon_facge, AddRetType | Add1ArgType), 2261 NEONMAP1(vcages_f32, aarch64_neon_facge, AddRetType | Add1ArgType), 2262 NEONMAP1(vcagtd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType), 2263 NEONMAP1(vcagts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType), 2264 NEONMAP1(vcaled_f64, aarch64_neon_facge, AddRetType | Add1ArgType), 2265 NEONMAP1(vcales_f32, aarch64_neon_facge, AddRetType | Add1ArgType), 2266 NEONMAP1(vcaltd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType), 2267 NEONMAP1(vcalts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType), 2268 NEONMAP1(vcvtad_s64_f64, aarch64_neon_fcvtas, AddRetType | Add1ArgType), 2269 NEONMAP1(vcvtad_u64_f64, aarch64_neon_fcvtau, AddRetType | Add1ArgType), 2270 NEONMAP1(vcvtas_s32_f32, aarch64_neon_fcvtas, AddRetType | Add1ArgType), 2271 NEONMAP1(vcvtas_u32_f32, aarch64_neon_fcvtau, AddRetType | Add1ArgType), 2272 NEONMAP1(vcvtd_n_f64_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType), 2273 NEONMAP1(vcvtd_n_f64_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType), 2274 NEONMAP1(vcvtd_n_s64_f64, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType), 2275 NEONMAP1(vcvtd_n_u64_f64, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType), 2276 NEONMAP1(vcvtmd_s64_f64, aarch64_neon_fcvtms, AddRetType | Add1ArgType), 2277 NEONMAP1(vcvtmd_u64_f64, aarch64_neon_fcvtmu, AddRetType | Add1ArgType), 2278 NEONMAP1(vcvtms_s32_f32, aarch64_neon_fcvtms, AddRetType | Add1ArgType), 2279 NEONMAP1(vcvtms_u32_f32, aarch64_neon_fcvtmu, AddRetType | Add1ArgType), 2280 NEONMAP1(vcvtnd_s64_f64, aarch64_neon_fcvtns, AddRetType | Add1ArgType), 2281 NEONMAP1(vcvtnd_u64_f64, aarch64_neon_fcvtnu, AddRetType | Add1ArgType), 2282 NEONMAP1(vcvtns_s32_f32, aarch64_neon_fcvtns, AddRetType | Add1ArgType), 2283 NEONMAP1(vcvtns_u32_f32, aarch64_neon_fcvtnu, AddRetType | Add1ArgType), 2284 NEONMAP1(vcvtpd_s64_f64, aarch64_neon_fcvtps, AddRetType | Add1ArgType), 2285 NEONMAP1(vcvtpd_u64_f64, aarch64_neon_fcvtpu, AddRetType | Add1ArgType), 2286 NEONMAP1(vcvtps_s32_f32, aarch64_neon_fcvtps, AddRetType | Add1ArgType), 2287 NEONMAP1(vcvtps_u32_f32, aarch64_neon_fcvtpu, AddRetType | Add1ArgType), 2288 NEONMAP1(vcvts_n_f32_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType), 2289 NEONMAP1(vcvts_n_f32_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType), 2290 NEONMAP1(vcvts_n_s32_f32, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType), 2291 NEONMAP1(vcvts_n_u32_f32, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType), 2292 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0), 2293 NEONMAP1(vmaxnmv_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 2294 NEONMAP1(vmaxnmvq_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 2295 NEONMAP1(vmaxnmvq_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 2296 NEONMAP1(vmaxv_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 2297 NEONMAP1(vmaxv_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType), 2298 NEONMAP1(vmaxv_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType), 2299 NEONMAP1(vmaxvq_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 2300 NEONMAP1(vmaxvq_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 2301 NEONMAP1(vmaxvq_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType), 2302 NEONMAP1(vmaxvq_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType), 2303 NEONMAP1(vminnmv_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 2304 NEONMAP1(vminnmvq_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 2305 NEONMAP1(vminnmvq_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 2306 NEONMAP1(vminv_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 2307 NEONMAP1(vminv_s32, aarch64_neon_sminv, AddRetType | Add1ArgType), 2308 NEONMAP1(vminv_u32, aarch64_neon_uminv, AddRetType | Add1ArgType), 2309 NEONMAP1(vminvq_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 2310 NEONMAP1(vminvq_f64, aarch64_neon_fminv, AddRetType | Add1ArgType), 2311 NEONMAP1(vminvq_s32, aarch64_neon_sminv, AddRetType | Add1ArgType), 2312 NEONMAP1(vminvq_u32, aarch64_neon_uminv, AddRetType | Add1ArgType), 2313 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0), 2314 NEONMAP1(vmulxd_f64, aarch64_neon_fmulx, Add1ArgType), 2315 NEONMAP1(vmulxs_f32, aarch64_neon_fmulx, Add1ArgType), 2316 NEONMAP1(vpaddd_s64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 2317 NEONMAP1(vpaddd_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 2318 NEONMAP1(vpmaxnmqd_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 2319 NEONMAP1(vpmaxnms_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 2320 NEONMAP1(vpmaxqd_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 2321 NEONMAP1(vpmaxs_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 2322 NEONMAP1(vpminnmqd_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 2323 NEONMAP1(vpminnms_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 2324 NEONMAP1(vpminqd_f64, aarch64_neon_fminv, AddRetType | Add1ArgType), 2325 NEONMAP1(vpmins_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 2326 NEONMAP1(vqabsb_s8, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors), 2327 NEONMAP1(vqabsd_s64, aarch64_neon_sqabs, Add1ArgType), 2328 NEONMAP1(vqabsh_s16, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors), 2329 NEONMAP1(vqabss_s32, aarch64_neon_sqabs, Add1ArgType), 2330 NEONMAP1(vqaddb_s8, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors), 2331 NEONMAP1(vqaddb_u8, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors), 2332 NEONMAP1(vqaddd_s64, aarch64_neon_sqadd, Add1ArgType), 2333 NEONMAP1(vqaddd_u64, aarch64_neon_uqadd, Add1ArgType), 2334 NEONMAP1(vqaddh_s16, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors), 2335 NEONMAP1(vqaddh_u16, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors), 2336 NEONMAP1(vqadds_s32, aarch64_neon_sqadd, Add1ArgType), 2337 NEONMAP1(vqadds_u32, aarch64_neon_uqadd, Add1ArgType), 2338 NEONMAP1(vqdmulhh_s16, aarch64_neon_sqdmulh, Vectorize1ArgType | Use64BitVectors), 2339 NEONMAP1(vqdmulhs_s32, aarch64_neon_sqdmulh, Add1ArgType), 2340 NEONMAP1(vqdmullh_s16, aarch64_neon_sqdmull, VectorRet | Use128BitVectors), 2341 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0), 2342 NEONMAP1(vqmovnd_s64, aarch64_neon_scalar_sqxtn, AddRetType | Add1ArgType), 2343 NEONMAP1(vqmovnd_u64, aarch64_neon_scalar_uqxtn, AddRetType | Add1ArgType), 2344 NEONMAP1(vqmovnh_s16, aarch64_neon_sqxtn, VectorRet | Use64BitVectors), 2345 NEONMAP1(vqmovnh_u16, aarch64_neon_uqxtn, VectorRet | Use64BitVectors), 2346 NEONMAP1(vqmovns_s32, aarch64_neon_sqxtn, VectorRet | Use64BitVectors), 2347 NEONMAP1(vqmovns_u32, aarch64_neon_uqxtn, VectorRet | Use64BitVectors), 2348 NEONMAP1(vqmovund_s64, aarch64_neon_scalar_sqxtun, AddRetType | Add1ArgType), 2349 NEONMAP1(vqmovunh_s16, aarch64_neon_sqxtun, VectorRet | Use64BitVectors), 2350 NEONMAP1(vqmovuns_s32, aarch64_neon_sqxtun, VectorRet | Use64BitVectors), 2351 NEONMAP1(vqnegb_s8, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors), 2352 NEONMAP1(vqnegd_s64, aarch64_neon_sqneg, Add1ArgType), 2353 NEONMAP1(vqnegh_s16, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors), 2354 NEONMAP1(vqnegs_s32, aarch64_neon_sqneg, Add1ArgType), 2355 NEONMAP1(vqrdmulhh_s16, aarch64_neon_sqrdmulh, Vectorize1ArgType | Use64BitVectors), 2356 NEONMAP1(vqrdmulhs_s32, aarch64_neon_sqrdmulh, Add1ArgType), 2357 NEONMAP1(vqrshlb_s8, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors), 2358 NEONMAP1(vqrshlb_u8, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors), 2359 NEONMAP1(vqrshld_s64, aarch64_neon_sqrshl, Add1ArgType), 2360 NEONMAP1(vqrshld_u64, aarch64_neon_uqrshl, Add1ArgType), 2361 NEONMAP1(vqrshlh_s16, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors), 2362 NEONMAP1(vqrshlh_u16, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors), 2363 NEONMAP1(vqrshls_s32, aarch64_neon_sqrshl, Add1ArgType), 2364 NEONMAP1(vqrshls_u32, aarch64_neon_uqrshl, Add1ArgType), 2365 NEONMAP1(vqrshrnd_n_s64, aarch64_neon_sqrshrn, AddRetType), 2366 NEONMAP1(vqrshrnd_n_u64, aarch64_neon_uqrshrn, AddRetType), 2367 NEONMAP1(vqrshrnh_n_s16, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors), 2368 NEONMAP1(vqrshrnh_n_u16, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors), 2369 NEONMAP1(vqrshrns_n_s32, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors), 2370 NEONMAP1(vqrshrns_n_u32, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors), 2371 NEONMAP1(vqrshrund_n_s64, aarch64_neon_sqrshrun, AddRetType), 2372 NEONMAP1(vqrshrunh_n_s16, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors), 2373 NEONMAP1(vqrshruns_n_s32, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors), 2374 NEONMAP1(vqshlb_n_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 2375 NEONMAP1(vqshlb_n_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 2376 NEONMAP1(vqshlb_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 2377 NEONMAP1(vqshlb_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 2378 NEONMAP1(vqshld_s64, aarch64_neon_sqshl, Add1ArgType), 2379 NEONMAP1(vqshld_u64, aarch64_neon_uqshl, Add1ArgType), 2380 NEONMAP1(vqshlh_n_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 2381 NEONMAP1(vqshlh_n_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 2382 NEONMAP1(vqshlh_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 2383 NEONMAP1(vqshlh_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 2384 NEONMAP1(vqshls_n_s32, aarch64_neon_sqshl, Add1ArgType), 2385 NEONMAP1(vqshls_n_u32, aarch64_neon_uqshl, Add1ArgType), 2386 NEONMAP1(vqshls_s32, aarch64_neon_sqshl, Add1ArgType), 2387 NEONMAP1(vqshls_u32, aarch64_neon_uqshl, Add1ArgType), 2388 NEONMAP1(vqshlub_n_s8, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors), 2389 NEONMAP1(vqshluh_n_s16, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors), 2390 NEONMAP1(vqshlus_n_s32, aarch64_neon_sqshlu, Add1ArgType), 2391 NEONMAP1(vqshrnd_n_s64, aarch64_neon_sqshrn, AddRetType), 2392 NEONMAP1(vqshrnd_n_u64, aarch64_neon_uqshrn, AddRetType), 2393 NEONMAP1(vqshrnh_n_s16, aarch64_neon_sqshrn, VectorRet | Use64BitVectors), 2394 NEONMAP1(vqshrnh_n_u16, aarch64_neon_uqshrn, VectorRet | Use64BitVectors), 2395 NEONMAP1(vqshrns_n_s32, aarch64_neon_sqshrn, VectorRet | Use64BitVectors), 2396 NEONMAP1(vqshrns_n_u32, aarch64_neon_uqshrn, VectorRet | Use64BitVectors), 2397 NEONMAP1(vqshrund_n_s64, aarch64_neon_sqshrun, AddRetType), 2398 NEONMAP1(vqshrunh_n_s16, aarch64_neon_sqshrun, VectorRet | Use64BitVectors), 2399 NEONMAP1(vqshruns_n_s32, aarch64_neon_sqshrun, VectorRet | Use64BitVectors), 2400 NEONMAP1(vqsubb_s8, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors), 2401 NEONMAP1(vqsubb_u8, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors), 2402 NEONMAP1(vqsubd_s64, aarch64_neon_sqsub, Add1ArgType), 2403 NEONMAP1(vqsubd_u64, aarch64_neon_uqsub, Add1ArgType), 2404 NEONMAP1(vqsubh_s16, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors), 2405 NEONMAP1(vqsubh_u16, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors), 2406 NEONMAP1(vqsubs_s32, aarch64_neon_sqsub, Add1ArgType), 2407 NEONMAP1(vqsubs_u32, aarch64_neon_uqsub, Add1ArgType), 2408 NEONMAP1(vrecped_f64, aarch64_neon_frecpe, Add1ArgType), 2409 NEONMAP1(vrecpes_f32, aarch64_neon_frecpe, Add1ArgType), 2410 NEONMAP1(vrecpxd_f64, aarch64_neon_frecpx, Add1ArgType), 2411 NEONMAP1(vrecpxs_f32, aarch64_neon_frecpx, Add1ArgType), 2412 NEONMAP1(vrshld_s64, aarch64_neon_srshl, Add1ArgType), 2413 NEONMAP1(vrshld_u64, aarch64_neon_urshl, Add1ArgType), 2414 NEONMAP1(vrsqrted_f64, aarch64_neon_frsqrte, Add1ArgType), 2415 NEONMAP1(vrsqrtes_f32, aarch64_neon_frsqrte, Add1ArgType), 2416 NEONMAP1(vrsqrtsd_f64, aarch64_neon_frsqrts, Add1ArgType), 2417 NEONMAP1(vrsqrtss_f32, aarch64_neon_frsqrts, Add1ArgType), 2418 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0), 2419 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0), 2420 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0), 2421 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0), 2422 NEONMAP1(vshld_s64, aarch64_neon_sshl, Add1ArgType), 2423 NEONMAP1(vshld_u64, aarch64_neon_ushl, Add1ArgType), 2424 NEONMAP1(vslid_n_s64, aarch64_neon_vsli, Vectorize1ArgType), 2425 NEONMAP1(vslid_n_u64, aarch64_neon_vsli, Vectorize1ArgType), 2426 NEONMAP1(vsqaddb_u8, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors), 2427 NEONMAP1(vsqaddd_u64, aarch64_neon_usqadd, Add1ArgType), 2428 NEONMAP1(vsqaddh_u16, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors), 2429 NEONMAP1(vsqadds_u32, aarch64_neon_usqadd, Add1ArgType), 2430 NEONMAP1(vsrid_n_s64, aarch64_neon_vsri, Vectorize1ArgType), 2431 NEONMAP1(vsrid_n_u64, aarch64_neon_vsri, Vectorize1ArgType), 2432 NEONMAP1(vuqaddb_s8, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors), 2433 NEONMAP1(vuqaddd_s64, aarch64_neon_suqadd, Add1ArgType), 2434 NEONMAP1(vuqaddh_s16, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors), 2435 NEONMAP1(vuqadds_s32, aarch64_neon_suqadd, Add1ArgType), 2436 }; 2437 2438 #undef NEONMAP0 2439 #undef NEONMAP1 2440 #undef NEONMAP2 2441 2442 static bool NEONSIMDIntrinsicsProvenSorted = false; 2443 2444 static bool AArch64SIMDIntrinsicsProvenSorted = false; 2445 static bool AArch64SISDIntrinsicsProvenSorted = false; 2446 2447 2448 static const NeonIntrinsicInfo * 2449 findNeonIntrinsicInMap(ArrayRef<NeonIntrinsicInfo> IntrinsicMap, 2450 unsigned BuiltinID, bool &MapProvenSorted) { 2451 2452 #ifndef NDEBUG 2453 if (!MapProvenSorted) { 2454 // FIXME: use std::is_sorted once C++11 is allowed 2455 for (unsigned i = 0; i < IntrinsicMap.size() - 1; ++i) 2456 assert(IntrinsicMap[i].BuiltinID <= IntrinsicMap[i + 1].BuiltinID); 2457 MapProvenSorted = true; 2458 } 2459 #endif 2460 2461 const NeonIntrinsicInfo *Builtin = 2462 std::lower_bound(IntrinsicMap.begin(), IntrinsicMap.end(), BuiltinID); 2463 2464 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID) 2465 return Builtin; 2466 2467 return nullptr; 2468 } 2469 2470 Function *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID, 2471 unsigned Modifier, 2472 llvm::Type *ArgType, 2473 const CallExpr *E) { 2474 int VectorSize = 0; 2475 if (Modifier & Use64BitVectors) 2476 VectorSize = 64; 2477 else if (Modifier & Use128BitVectors) 2478 VectorSize = 128; 2479 2480 // Return type. 2481 SmallVector<llvm::Type *, 3> Tys; 2482 if (Modifier & AddRetType) { 2483 llvm::Type *Ty = ConvertType(E->getCallReturnType()); 2484 if (Modifier & VectorizeRetType) 2485 Ty = llvm::VectorType::get( 2486 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1); 2487 2488 Tys.push_back(Ty); 2489 } 2490 2491 // Arguments. 2492 if (Modifier & VectorizeArgTypes) { 2493 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1; 2494 ArgType = llvm::VectorType::get(ArgType, Elts); 2495 } 2496 2497 if (Modifier & (Add1ArgType | Add2ArgTypes)) 2498 Tys.push_back(ArgType); 2499 2500 if (Modifier & Add2ArgTypes) 2501 Tys.push_back(ArgType); 2502 2503 if (Modifier & InventFloatType) 2504 Tys.push_back(FloatTy); 2505 2506 return CGM.getIntrinsic(IntrinsicID, Tys); 2507 } 2508 2509 static Value *EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, 2510 const NeonIntrinsicInfo &SISDInfo, 2511 SmallVectorImpl<Value *> &Ops, 2512 const CallExpr *E) { 2513 unsigned BuiltinID = SISDInfo.BuiltinID; 2514 unsigned int Int = SISDInfo.LLVMIntrinsic; 2515 unsigned Modifier = SISDInfo.TypeModifier; 2516 const char *s = SISDInfo.NameHint; 2517 2518 switch (BuiltinID) { 2519 case NEON::BI__builtin_neon_vcled_s64: 2520 case NEON::BI__builtin_neon_vcled_u64: 2521 case NEON::BI__builtin_neon_vcles_f32: 2522 case NEON::BI__builtin_neon_vcled_f64: 2523 case NEON::BI__builtin_neon_vcltd_s64: 2524 case NEON::BI__builtin_neon_vcltd_u64: 2525 case NEON::BI__builtin_neon_vclts_f32: 2526 case NEON::BI__builtin_neon_vcltd_f64: 2527 case NEON::BI__builtin_neon_vcales_f32: 2528 case NEON::BI__builtin_neon_vcaled_f64: 2529 case NEON::BI__builtin_neon_vcalts_f32: 2530 case NEON::BI__builtin_neon_vcaltd_f64: 2531 // Only one direction of comparisons actually exist, cmle is actually a cmge 2532 // with swapped operands. The table gives us the right intrinsic but we 2533 // still need to do the swap. 2534 std::swap(Ops[0], Ops[1]); 2535 break; 2536 } 2537 2538 assert(Int && "Generic code assumes a valid intrinsic"); 2539 2540 // Determine the type(s) of this overloaded AArch64 intrinsic. 2541 const Expr *Arg = E->getArg(0); 2542 llvm::Type *ArgTy = CGF.ConvertType(Arg->getType()); 2543 Function *F = CGF.LookupNeonLLVMIntrinsic(Int, Modifier, ArgTy, E); 2544 2545 int j = 0; 2546 ConstantInt *C0 = ConstantInt::get(CGF.SizeTy, 0); 2547 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end(); 2548 ai != ae; ++ai, ++j) { 2549 llvm::Type *ArgTy = ai->getType(); 2550 if (Ops[j]->getType()->getPrimitiveSizeInBits() == 2551 ArgTy->getPrimitiveSizeInBits()) 2552 continue; 2553 2554 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy()); 2555 // The constant argument to an _n_ intrinsic always has Int32Ty, so truncate 2556 // it before inserting. 2557 Ops[j] = 2558 CGF.Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType()); 2559 Ops[j] = 2560 CGF.Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0); 2561 } 2562 2563 Value *Result = CGF.EmitNeonCall(F, Ops, s); 2564 llvm::Type *ResultType = CGF.ConvertType(E->getType()); 2565 if (ResultType->getPrimitiveSizeInBits() < 2566 Result->getType()->getPrimitiveSizeInBits()) 2567 return CGF.Builder.CreateExtractElement(Result, C0); 2568 2569 return CGF.Builder.CreateBitCast(Result, ResultType, s); 2570 } 2571 2572 Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( 2573 unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, 2574 const char *NameHint, unsigned Modifier, const CallExpr *E, 2575 SmallVectorImpl<llvm::Value *> &Ops, llvm::Value *Align) { 2576 // Get the last argument, which specifies the vector type. 2577 llvm::APSInt NeonTypeConst; 2578 const Expr *Arg = E->getArg(E->getNumArgs() - 1); 2579 if (!Arg->isIntegerConstantExpr(NeonTypeConst, getContext())) 2580 return nullptr; 2581 2582 // Determine the type of this overloaded NEON intrinsic. 2583 NeonTypeFlags Type(NeonTypeConst.getZExtValue()); 2584 bool Usgn = Type.isUnsigned(); 2585 bool Quad = Type.isQuad(); 2586 2587 llvm::VectorType *VTy = GetNeonType(this, Type); 2588 llvm::Type *Ty = VTy; 2589 if (!Ty) 2590 return nullptr; 2591 2592 unsigned Int = LLVMIntrinsic; 2593 if ((Modifier & UnsignedAlts) && !Usgn) 2594 Int = AltLLVMIntrinsic; 2595 2596 switch (BuiltinID) { 2597 default: break; 2598 case NEON::BI__builtin_neon_vabs_v: 2599 case NEON::BI__builtin_neon_vabsq_v: 2600 if (VTy->getElementType()->isFloatingPointTy()) 2601 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs"); 2602 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs"); 2603 case NEON::BI__builtin_neon_vaddhn_v: { 2604 llvm::VectorType *SrcTy = 2605 llvm::VectorType::getExtendedElementVectorType(VTy); 2606 2607 // %sum = add <4 x i32> %lhs, %rhs 2608 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 2609 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); 2610 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn"); 2611 2612 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16> 2613 Constant *ShiftAmt = ConstantInt::get(SrcTy->getElementType(), 2614 SrcTy->getScalarSizeInBits() / 2); 2615 ShiftAmt = ConstantVector::getSplat(VTy->getNumElements(), ShiftAmt); 2616 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn"); 2617 2618 // %res = trunc <4 x i32> %high to <4 x i16> 2619 return Builder.CreateTrunc(Ops[0], VTy, "vaddhn"); 2620 } 2621 case NEON::BI__builtin_neon_vcale_v: 2622 case NEON::BI__builtin_neon_vcaleq_v: 2623 case NEON::BI__builtin_neon_vcalt_v: 2624 case NEON::BI__builtin_neon_vcaltq_v: 2625 std::swap(Ops[0], Ops[1]); 2626 case NEON::BI__builtin_neon_vcage_v: 2627 case NEON::BI__builtin_neon_vcageq_v: 2628 case NEON::BI__builtin_neon_vcagt_v: 2629 case NEON::BI__builtin_neon_vcagtq_v: { 2630 llvm::Type *VecFlt = llvm::VectorType::get( 2631 VTy->getScalarSizeInBits() == 32 ? FloatTy : DoubleTy, 2632 VTy->getNumElements()); 2633 llvm::Type *Tys[] = { VTy, VecFlt }; 2634 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 2635 return EmitNeonCall(F, Ops, NameHint); 2636 } 2637 case NEON::BI__builtin_neon_vclz_v: 2638 case NEON::BI__builtin_neon_vclzq_v: 2639 // We generate target-independent intrinsic, which needs a second argument 2640 // for whether or not clz of zero is undefined; on ARM it isn't. 2641 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef())); 2642 break; 2643 case NEON::BI__builtin_neon_vcvt_f32_v: 2644 case NEON::BI__builtin_neon_vcvtq_f32_v: 2645 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 2646 Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, Quad)); 2647 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") 2648 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); 2649 case NEON::BI__builtin_neon_vcvt_n_f32_v: 2650 case NEON::BI__builtin_neon_vcvt_n_f64_v: 2651 case NEON::BI__builtin_neon_vcvtq_n_f32_v: 2652 case NEON::BI__builtin_neon_vcvtq_n_f64_v: { 2653 bool Double = 2654 (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 2655 llvm::Type *FloatTy = 2656 GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64 2657 : NeonTypeFlags::Float32, 2658 false, Quad)); 2659 llvm::Type *Tys[2] = { FloatTy, Ty }; 2660 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic; 2661 Function *F = CGM.getIntrinsic(Int, Tys); 2662 return EmitNeonCall(F, Ops, "vcvt_n"); 2663 } 2664 case NEON::BI__builtin_neon_vcvt_n_s32_v: 2665 case NEON::BI__builtin_neon_vcvt_n_u32_v: 2666 case NEON::BI__builtin_neon_vcvt_n_s64_v: 2667 case NEON::BI__builtin_neon_vcvt_n_u64_v: 2668 case NEON::BI__builtin_neon_vcvtq_n_s32_v: 2669 case NEON::BI__builtin_neon_vcvtq_n_u32_v: 2670 case NEON::BI__builtin_neon_vcvtq_n_s64_v: 2671 case NEON::BI__builtin_neon_vcvtq_n_u64_v: { 2672 bool Double = 2673 (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 2674 llvm::Type *FloatTy = 2675 GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64 2676 : NeonTypeFlags::Float32, 2677 false, Quad)); 2678 llvm::Type *Tys[2] = { Ty, FloatTy }; 2679 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 2680 return EmitNeonCall(F, Ops, "vcvt_n"); 2681 } 2682 case NEON::BI__builtin_neon_vcvt_s32_v: 2683 case NEON::BI__builtin_neon_vcvt_u32_v: 2684 case NEON::BI__builtin_neon_vcvt_s64_v: 2685 case NEON::BI__builtin_neon_vcvt_u64_v: 2686 case NEON::BI__builtin_neon_vcvtq_s32_v: 2687 case NEON::BI__builtin_neon_vcvtq_u32_v: 2688 case NEON::BI__builtin_neon_vcvtq_s64_v: 2689 case NEON::BI__builtin_neon_vcvtq_u64_v: { 2690 bool Double = 2691 (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 2692 llvm::Type *FloatTy = 2693 GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64 2694 : NeonTypeFlags::Float32, 2695 false, Quad)); 2696 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy); 2697 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt") 2698 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt"); 2699 } 2700 case NEON::BI__builtin_neon_vcvta_s32_v: 2701 case NEON::BI__builtin_neon_vcvta_s64_v: 2702 case NEON::BI__builtin_neon_vcvta_u32_v: 2703 case NEON::BI__builtin_neon_vcvta_u64_v: 2704 case NEON::BI__builtin_neon_vcvtaq_s32_v: 2705 case NEON::BI__builtin_neon_vcvtaq_s64_v: 2706 case NEON::BI__builtin_neon_vcvtaq_u32_v: 2707 case NEON::BI__builtin_neon_vcvtaq_u64_v: 2708 case NEON::BI__builtin_neon_vcvtn_s32_v: 2709 case NEON::BI__builtin_neon_vcvtn_s64_v: 2710 case NEON::BI__builtin_neon_vcvtn_u32_v: 2711 case NEON::BI__builtin_neon_vcvtn_u64_v: 2712 case NEON::BI__builtin_neon_vcvtnq_s32_v: 2713 case NEON::BI__builtin_neon_vcvtnq_s64_v: 2714 case NEON::BI__builtin_neon_vcvtnq_u32_v: 2715 case NEON::BI__builtin_neon_vcvtnq_u64_v: 2716 case NEON::BI__builtin_neon_vcvtp_s32_v: 2717 case NEON::BI__builtin_neon_vcvtp_s64_v: 2718 case NEON::BI__builtin_neon_vcvtp_u32_v: 2719 case NEON::BI__builtin_neon_vcvtp_u64_v: 2720 case NEON::BI__builtin_neon_vcvtpq_s32_v: 2721 case NEON::BI__builtin_neon_vcvtpq_s64_v: 2722 case NEON::BI__builtin_neon_vcvtpq_u32_v: 2723 case NEON::BI__builtin_neon_vcvtpq_u64_v: 2724 case NEON::BI__builtin_neon_vcvtm_s32_v: 2725 case NEON::BI__builtin_neon_vcvtm_s64_v: 2726 case NEON::BI__builtin_neon_vcvtm_u32_v: 2727 case NEON::BI__builtin_neon_vcvtm_u64_v: 2728 case NEON::BI__builtin_neon_vcvtmq_s32_v: 2729 case NEON::BI__builtin_neon_vcvtmq_s64_v: 2730 case NEON::BI__builtin_neon_vcvtmq_u32_v: 2731 case NEON::BI__builtin_neon_vcvtmq_u64_v: { 2732 bool Double = 2733 (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 2734 llvm::Type *InTy = 2735 GetNeonType(this, 2736 NeonTypeFlags(Double ? NeonTypeFlags::Float64 2737 : NeonTypeFlags::Float32, false, Quad)); 2738 llvm::Type *Tys[2] = { Ty, InTy }; 2739 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint); 2740 } 2741 case NEON::BI__builtin_neon_vext_v: 2742 case NEON::BI__builtin_neon_vextq_v: { 2743 int CV = cast<ConstantInt>(Ops[2])->getSExtValue(); 2744 SmallVector<Constant*, 16> Indices; 2745 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 2746 Indices.push_back(ConstantInt::get(Int32Ty, i+CV)); 2747 2748 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 2749 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 2750 Value *SV = llvm::ConstantVector::get(Indices); 2751 return Builder.CreateShuffleVector(Ops[0], Ops[1], SV, "vext"); 2752 } 2753 case NEON::BI__builtin_neon_vfma_v: 2754 case NEON::BI__builtin_neon_vfmaq_v: { 2755 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 2756 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 2757 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 2758 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 2759 2760 // NEON intrinsic puts accumulator first, unlike the LLVM fma. 2761 return Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]); 2762 } 2763 case NEON::BI__builtin_neon_vld1_v: 2764 case NEON::BI__builtin_neon_vld1q_v: 2765 Ops.push_back(Align); 2766 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vld1"); 2767 case NEON::BI__builtin_neon_vld2_v: 2768 case NEON::BI__builtin_neon_vld2q_v: 2769 case NEON::BI__builtin_neon_vld3_v: 2770 case NEON::BI__builtin_neon_vld3q_v: 2771 case NEON::BI__builtin_neon_vld4_v: 2772 case NEON::BI__builtin_neon_vld4q_v: { 2773 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Ty); 2774 Ops[1] = Builder.CreateCall2(F, Ops[1], Align, NameHint); 2775 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 2776 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 2777 return Builder.CreateStore(Ops[1], Ops[0]); 2778 } 2779 case NEON::BI__builtin_neon_vld1_dup_v: 2780 case NEON::BI__builtin_neon_vld1q_dup_v: { 2781 Value *V = UndefValue::get(Ty); 2782 Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 2783 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 2784 LoadInst *Ld = Builder.CreateLoad(Ops[0]); 2785 Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue()); 2786 llvm::Constant *CI = ConstantInt::get(SizeTy, 0); 2787 Ops[0] = Builder.CreateInsertElement(V, Ld, CI); 2788 return EmitNeonSplat(Ops[0], CI); 2789 } 2790 case NEON::BI__builtin_neon_vld2_lane_v: 2791 case NEON::BI__builtin_neon_vld2q_lane_v: 2792 case NEON::BI__builtin_neon_vld3_lane_v: 2793 case NEON::BI__builtin_neon_vld3q_lane_v: 2794 case NEON::BI__builtin_neon_vld4_lane_v: 2795 case NEON::BI__builtin_neon_vld4q_lane_v: { 2796 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Ty); 2797 for (unsigned I = 2; I < Ops.size() - 1; ++I) 2798 Ops[I] = Builder.CreateBitCast(Ops[I], Ty); 2799 Ops.push_back(Align); 2800 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint); 2801 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 2802 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 2803 return Builder.CreateStore(Ops[1], Ops[0]); 2804 } 2805 case NEON::BI__builtin_neon_vmovl_v: { 2806 llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy); 2807 Ops[0] = Builder.CreateBitCast(Ops[0], DTy); 2808 if (Usgn) 2809 return Builder.CreateZExt(Ops[0], Ty, "vmovl"); 2810 return Builder.CreateSExt(Ops[0], Ty, "vmovl"); 2811 } 2812 case NEON::BI__builtin_neon_vmovn_v: { 2813 llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy); 2814 Ops[0] = Builder.CreateBitCast(Ops[0], QTy); 2815 return Builder.CreateTrunc(Ops[0], Ty, "vmovn"); 2816 } 2817 case NEON::BI__builtin_neon_vmull_v: 2818 // FIXME: the integer vmull operations could be emitted in terms of pure 2819 // LLVM IR (2 exts followed by a mul). Unfortunately LLVM has a habit of 2820 // hoisting the exts outside loops. Until global ISel comes along that can 2821 // see through such movement this leads to bad CodeGen. So we need an 2822 // intrinsic for now. 2823 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls; 2824 Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int; 2825 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); 2826 case NEON::BI__builtin_neon_vpadal_v: 2827 case NEON::BI__builtin_neon_vpadalq_v: { 2828 // The source operand type has twice as many elements of half the size. 2829 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); 2830 llvm::Type *EltTy = 2831 llvm::IntegerType::get(getLLVMContext(), EltBits / 2); 2832 llvm::Type *NarrowTy = 2833 llvm::VectorType::get(EltTy, VTy->getNumElements() * 2); 2834 llvm::Type *Tys[2] = { Ty, NarrowTy }; 2835 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); 2836 } 2837 case NEON::BI__builtin_neon_vpaddl_v: 2838 case NEON::BI__builtin_neon_vpaddlq_v: { 2839 // The source operand type has twice as many elements of half the size. 2840 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); 2841 llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2); 2842 llvm::Type *NarrowTy = 2843 llvm::VectorType::get(EltTy, VTy->getNumElements() * 2); 2844 llvm::Type *Tys[2] = { Ty, NarrowTy }; 2845 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl"); 2846 } 2847 case NEON::BI__builtin_neon_vqdmlal_v: 2848 case NEON::BI__builtin_neon_vqdmlsl_v: { 2849 SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end()); 2850 Value *Mul = EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), 2851 MulOps, "vqdmlal"); 2852 2853 SmallVector<Value *, 2> AccumOps; 2854 AccumOps.push_back(Ops[0]); 2855 AccumOps.push_back(Mul); 2856 return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), 2857 AccumOps, NameHint); 2858 } 2859 case NEON::BI__builtin_neon_vqshl_n_v: 2860 case NEON::BI__builtin_neon_vqshlq_n_v: 2861 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n", 2862 1, false); 2863 case NEON::BI__builtin_neon_vqshlu_n_v: 2864 case NEON::BI__builtin_neon_vqshluq_n_v: 2865 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n", 2866 1, false); 2867 case NEON::BI__builtin_neon_vrecpe_v: 2868 case NEON::BI__builtin_neon_vrecpeq_v: 2869 case NEON::BI__builtin_neon_vrsqrte_v: 2870 case NEON::BI__builtin_neon_vrsqrteq_v: 2871 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic; 2872 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint); 2873 2874 case NEON::BI__builtin_neon_vrshr_n_v: 2875 case NEON::BI__builtin_neon_vrshrq_n_v: 2876 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", 2877 1, true); 2878 case NEON::BI__builtin_neon_vshl_n_v: 2879 case NEON::BI__builtin_neon_vshlq_n_v: 2880 Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false); 2881 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1], 2882 "vshl_n"); 2883 case NEON::BI__builtin_neon_vshll_n_v: { 2884 llvm::Type *SrcTy = llvm::VectorType::getTruncatedElementVectorType(VTy); 2885 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 2886 if (Usgn) 2887 Ops[0] = Builder.CreateZExt(Ops[0], VTy); 2888 else 2889 Ops[0] = Builder.CreateSExt(Ops[0], VTy); 2890 Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false); 2891 return Builder.CreateShl(Ops[0], Ops[1], "vshll_n"); 2892 } 2893 case NEON::BI__builtin_neon_vshrn_n_v: { 2894 llvm::Type *SrcTy = llvm::VectorType::getExtendedElementVectorType(VTy); 2895 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 2896 Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false); 2897 if (Usgn) 2898 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]); 2899 else 2900 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]); 2901 return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n"); 2902 } 2903 case NEON::BI__builtin_neon_vshr_n_v: 2904 case NEON::BI__builtin_neon_vshrq_n_v: 2905 return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n"); 2906 case NEON::BI__builtin_neon_vst1_v: 2907 case NEON::BI__builtin_neon_vst1q_v: 2908 case NEON::BI__builtin_neon_vst2_v: 2909 case NEON::BI__builtin_neon_vst2q_v: 2910 case NEON::BI__builtin_neon_vst3_v: 2911 case NEON::BI__builtin_neon_vst3q_v: 2912 case NEON::BI__builtin_neon_vst4_v: 2913 case NEON::BI__builtin_neon_vst4q_v: 2914 case NEON::BI__builtin_neon_vst2_lane_v: 2915 case NEON::BI__builtin_neon_vst2q_lane_v: 2916 case NEON::BI__builtin_neon_vst3_lane_v: 2917 case NEON::BI__builtin_neon_vst3q_lane_v: 2918 case NEON::BI__builtin_neon_vst4_lane_v: 2919 case NEON::BI__builtin_neon_vst4q_lane_v: 2920 Ops.push_back(Align); 2921 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, ""); 2922 case NEON::BI__builtin_neon_vsubhn_v: { 2923 llvm::VectorType *SrcTy = 2924 llvm::VectorType::getExtendedElementVectorType(VTy); 2925 2926 // %sum = add <4 x i32> %lhs, %rhs 2927 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 2928 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); 2929 Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn"); 2930 2931 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16> 2932 Constant *ShiftAmt = ConstantInt::get(SrcTy->getElementType(), 2933 SrcTy->getScalarSizeInBits() / 2); 2934 ShiftAmt = ConstantVector::getSplat(VTy->getNumElements(), ShiftAmt); 2935 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn"); 2936 2937 // %res = trunc <4 x i32> %high to <4 x i16> 2938 return Builder.CreateTrunc(Ops[0], VTy, "vsubhn"); 2939 } 2940 case NEON::BI__builtin_neon_vtrn_v: 2941 case NEON::BI__builtin_neon_vtrnq_v: { 2942 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 2943 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 2944 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 2945 Value *SV = nullptr; 2946 2947 for (unsigned vi = 0; vi != 2; ++vi) { 2948 SmallVector<Constant*, 16> Indices; 2949 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 2950 Indices.push_back(Builder.getInt32(i+vi)); 2951 Indices.push_back(Builder.getInt32(i+e+vi)); 2952 } 2953 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); 2954 SV = llvm::ConstantVector::get(Indices); 2955 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn"); 2956 SV = Builder.CreateStore(SV, Addr); 2957 } 2958 return SV; 2959 } 2960 case NEON::BI__builtin_neon_vtst_v: 2961 case NEON::BI__builtin_neon_vtstq_v: { 2962 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 2963 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 2964 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); 2965 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], 2966 ConstantAggregateZero::get(Ty)); 2967 return Builder.CreateSExt(Ops[0], Ty, "vtst"); 2968 } 2969 case NEON::BI__builtin_neon_vuzp_v: 2970 case NEON::BI__builtin_neon_vuzpq_v: { 2971 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 2972 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 2973 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 2974 Value *SV = nullptr; 2975 2976 for (unsigned vi = 0; vi != 2; ++vi) { 2977 SmallVector<Constant*, 16> Indices; 2978 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 2979 Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi)); 2980 2981 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); 2982 SV = llvm::ConstantVector::get(Indices); 2983 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp"); 2984 SV = Builder.CreateStore(SV, Addr); 2985 } 2986 return SV; 2987 } 2988 case NEON::BI__builtin_neon_vzip_v: 2989 case NEON::BI__builtin_neon_vzipq_v: { 2990 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 2991 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 2992 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 2993 Value *SV = nullptr; 2994 2995 for (unsigned vi = 0; vi != 2; ++vi) { 2996 SmallVector<Constant*, 16> Indices; 2997 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 2998 Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1)); 2999 Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e)); 3000 } 3001 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); 3002 SV = llvm::ConstantVector::get(Indices); 3003 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip"); 3004 SV = Builder.CreateStore(SV, Addr); 3005 } 3006 return SV; 3007 } 3008 } 3009 3010 assert(Int && "Expected valid intrinsic number"); 3011 3012 // Determine the type(s) of this overloaded AArch64 intrinsic. 3013 Function *F = LookupNeonLLVMIntrinsic(Int, Modifier, Ty, E); 3014 3015 Value *Result = EmitNeonCall(F, Ops, NameHint); 3016 llvm::Type *ResultType = ConvertType(E->getType()); 3017 // AArch64 intrinsic one-element vector type cast to 3018 // scalar type expected by the builtin 3019 return Builder.CreateBitCast(Result, ResultType, NameHint); 3020 } 3021 3022 Value *CodeGenFunction::EmitAArch64CompareBuiltinExpr( 3023 Value *Op, llvm::Type *Ty, const CmpInst::Predicate Fp, 3024 const CmpInst::Predicate Ip, const Twine &Name) { 3025 llvm::Type *OTy = Op->getType(); 3026 3027 // FIXME: this is utterly horrific. We should not be looking at previous 3028 // codegen context to find out what needs doing. Unfortunately TableGen 3029 // currently gives us exactly the same calls for vceqz_f32 and vceqz_s32 3030 // (etc). 3031 if (BitCastInst *BI = dyn_cast<BitCastInst>(Op)) 3032 OTy = BI->getOperand(0)->getType(); 3033 3034 Op = Builder.CreateBitCast(Op, OTy); 3035 if (OTy->getScalarType()->isFloatingPointTy()) { 3036 Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy)); 3037 } else { 3038 Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy)); 3039 } 3040 return Builder.CreateSExt(Op, Ty, Name); 3041 } 3042 3043 static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops, 3044 Value *ExtOp, Value *IndexOp, 3045 llvm::Type *ResTy, unsigned IntID, 3046 const char *Name) { 3047 SmallVector<Value *, 2> TblOps; 3048 if (ExtOp) 3049 TblOps.push_back(ExtOp); 3050 3051 // Build a vector containing sequential number like (0, 1, 2, ..., 15) 3052 SmallVector<Constant*, 16> Indices; 3053 llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType()); 3054 for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) { 3055 Indices.push_back(ConstantInt::get(CGF.Int32Ty, 2*i)); 3056 Indices.push_back(ConstantInt::get(CGF.Int32Ty, 2*i+1)); 3057 } 3058 Value *SV = llvm::ConstantVector::get(Indices); 3059 3060 int PairPos = 0, End = Ops.size() - 1; 3061 while (PairPos < End) { 3062 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], 3063 Ops[PairPos+1], SV, Name)); 3064 PairPos += 2; 3065 } 3066 3067 // If there's an odd number of 64-bit lookup table, fill the high 64-bit 3068 // of the 128-bit lookup table with zero. 3069 if (PairPos == End) { 3070 Value *ZeroTbl = ConstantAggregateZero::get(TblTy); 3071 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], 3072 ZeroTbl, SV, Name)); 3073 } 3074 3075 Function *TblF; 3076 TblOps.push_back(IndexOp); 3077 TblF = CGF.CGM.getIntrinsic(IntID, ResTy); 3078 3079 return CGF.EmitNeonCall(TblF, TblOps, Name); 3080 } 3081 3082 Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, 3083 const CallExpr *E) { 3084 unsigned HintID = static_cast<unsigned>(-1); 3085 switch (BuiltinID) { 3086 default: break; 3087 case ARM::BI__builtin_arm_nop: 3088 HintID = 0; 3089 break; 3090 case ARM::BI__builtin_arm_yield: 3091 case ARM::BI__yield: 3092 HintID = 1; 3093 break; 3094 case ARM::BI__builtin_arm_wfe: 3095 case ARM::BI__wfe: 3096 HintID = 2; 3097 break; 3098 case ARM::BI__builtin_arm_wfi: 3099 case ARM::BI__wfi: 3100 HintID = 3; 3101 break; 3102 case ARM::BI__builtin_arm_sev: 3103 case ARM::BI__sev: 3104 HintID = 4; 3105 break; 3106 case ARM::BI__builtin_arm_sevl: 3107 case ARM::BI__sevl: 3108 HintID = 5; 3109 break; 3110 } 3111 3112 if (HintID != static_cast<unsigned>(-1)) { 3113 Function *F = CGM.getIntrinsic(Intrinsic::arm_hint); 3114 return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID)); 3115 } 3116 3117 if (BuiltinID == ARM::BI__builtin_arm_dbg) { 3118 Value *Option = EmitScalarExpr(E->getArg(0)); 3119 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option); 3120 } 3121 3122 if (BuiltinID == ARM::BI__builtin_arm_prefetch) { 3123 Value *Address = EmitScalarExpr(E->getArg(0)); 3124 Value *RW = EmitScalarExpr(E->getArg(1)); 3125 Value *IsData = EmitScalarExpr(E->getArg(2)); 3126 3127 // Locality is not supported on ARM target 3128 Value *Locality = llvm::ConstantInt::get(Int32Ty, 3); 3129 3130 Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 3131 return Builder.CreateCall4(F, Address, RW, Locality, IsData); 3132 } 3133 3134 if (BuiltinID == ARM::BI__builtin_arm_rbit) { 3135 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_rbit), 3136 EmitScalarExpr(E->getArg(0)), 3137 "rbit"); 3138 } 3139 3140 if (BuiltinID == ARM::BI__clear_cache) { 3141 assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments"); 3142 const FunctionDecl *FD = E->getDirectCallee(); 3143 SmallVector<Value*, 2> Ops; 3144 for (unsigned i = 0; i < 2; i++) 3145 Ops.push_back(EmitScalarExpr(E->getArg(i))); 3146 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType()); 3147 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty); 3148 StringRef Name = FD->getName(); 3149 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); 3150 } 3151 3152 if (BuiltinID == ARM::BI__builtin_arm_ldrexd || 3153 ((BuiltinID == ARM::BI__builtin_arm_ldrex || 3154 BuiltinID == ARM::BI__builtin_arm_ldaex) && 3155 getContext().getTypeSize(E->getType()) == 64) || 3156 BuiltinID == ARM::BI__ldrexd) { 3157 Function *F; 3158 3159 switch (BuiltinID) { 3160 default: llvm_unreachable("unexpected builtin"); 3161 case ARM::BI__builtin_arm_ldaex: 3162 F = CGM.getIntrinsic(Intrinsic::arm_ldaexd); 3163 break; 3164 case ARM::BI__builtin_arm_ldrexd: 3165 case ARM::BI__builtin_arm_ldrex: 3166 case ARM::BI__ldrexd: 3167 F = CGM.getIntrinsic(Intrinsic::arm_ldrexd); 3168 break; 3169 } 3170 3171 Value *LdPtr = EmitScalarExpr(E->getArg(0)); 3172 Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy), 3173 "ldrexd"); 3174 3175 Value *Val0 = Builder.CreateExtractValue(Val, 1); 3176 Value *Val1 = Builder.CreateExtractValue(Val, 0); 3177 Val0 = Builder.CreateZExt(Val0, Int64Ty); 3178 Val1 = Builder.CreateZExt(Val1, Int64Ty); 3179 3180 Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32); 3181 Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */); 3182 Val = Builder.CreateOr(Val, Val1); 3183 return Builder.CreateBitCast(Val, ConvertType(E->getType())); 3184 } 3185 3186 if (BuiltinID == ARM::BI__builtin_arm_ldrex || 3187 BuiltinID == ARM::BI__builtin_arm_ldaex) { 3188 Value *LoadAddr = EmitScalarExpr(E->getArg(0)); 3189 3190 QualType Ty = E->getType(); 3191 llvm::Type *RealResTy = ConvertType(Ty); 3192 llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(), 3193 getContext().getTypeSize(Ty)); 3194 LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo()); 3195 3196 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_ldaex 3197 ? Intrinsic::arm_ldaex 3198 : Intrinsic::arm_ldrex, 3199 LoadAddr->getType()); 3200 Value *Val = Builder.CreateCall(F, LoadAddr, "ldrex"); 3201 3202 if (RealResTy->isPointerTy()) 3203 return Builder.CreateIntToPtr(Val, RealResTy); 3204 else { 3205 Val = Builder.CreateTruncOrBitCast(Val, IntResTy); 3206 return Builder.CreateBitCast(Val, RealResTy); 3207 } 3208 } 3209 3210 if (BuiltinID == ARM::BI__builtin_arm_strexd || 3211 ((BuiltinID == ARM::BI__builtin_arm_stlex || 3212 BuiltinID == ARM::BI__builtin_arm_strex) && 3213 getContext().getTypeSize(E->getArg(0)->getType()) == 64)) { 3214 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex 3215 ? Intrinsic::arm_stlexd 3216 : Intrinsic::arm_strexd); 3217 llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, NULL); 3218 3219 Value *Tmp = CreateMemTemp(E->getArg(0)->getType()); 3220 Value *Val = EmitScalarExpr(E->getArg(0)); 3221 Builder.CreateStore(Val, Tmp); 3222 3223 Value *LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy)); 3224 Val = Builder.CreateLoad(LdPtr); 3225 3226 Value *Arg0 = Builder.CreateExtractValue(Val, 0); 3227 Value *Arg1 = Builder.CreateExtractValue(Val, 1); 3228 Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), Int8PtrTy); 3229 return Builder.CreateCall3(F, Arg0, Arg1, StPtr, "strexd"); 3230 } 3231 3232 if (BuiltinID == ARM::BI__builtin_arm_strex || 3233 BuiltinID == ARM::BI__builtin_arm_stlex) { 3234 Value *StoreVal = EmitScalarExpr(E->getArg(0)); 3235 Value *StoreAddr = EmitScalarExpr(E->getArg(1)); 3236 3237 QualType Ty = E->getArg(0)->getType(); 3238 llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(), 3239 getContext().getTypeSize(Ty)); 3240 StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo()); 3241 3242 if (StoreVal->getType()->isPointerTy()) 3243 StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty); 3244 else { 3245 StoreVal = Builder.CreateBitCast(StoreVal, StoreTy); 3246 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty); 3247 } 3248 3249 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex 3250 ? Intrinsic::arm_stlex 3251 : Intrinsic::arm_strex, 3252 StoreAddr->getType()); 3253 return Builder.CreateCall2(F, StoreVal, StoreAddr, "strex"); 3254 } 3255 3256 if (BuiltinID == ARM::BI__builtin_arm_clrex) { 3257 Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex); 3258 return Builder.CreateCall(F); 3259 } 3260 3261 // CRC32 3262 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic; 3263 switch (BuiltinID) { 3264 case ARM::BI__builtin_arm_crc32b: 3265 CRCIntrinsicID = Intrinsic::arm_crc32b; break; 3266 case ARM::BI__builtin_arm_crc32cb: 3267 CRCIntrinsicID = Intrinsic::arm_crc32cb; break; 3268 case ARM::BI__builtin_arm_crc32h: 3269 CRCIntrinsicID = Intrinsic::arm_crc32h; break; 3270 case ARM::BI__builtin_arm_crc32ch: 3271 CRCIntrinsicID = Intrinsic::arm_crc32ch; break; 3272 case ARM::BI__builtin_arm_crc32w: 3273 case ARM::BI__builtin_arm_crc32d: 3274 CRCIntrinsicID = Intrinsic::arm_crc32w; break; 3275 case ARM::BI__builtin_arm_crc32cw: 3276 case ARM::BI__builtin_arm_crc32cd: 3277 CRCIntrinsicID = Intrinsic::arm_crc32cw; break; 3278 } 3279 3280 if (CRCIntrinsicID != Intrinsic::not_intrinsic) { 3281 Value *Arg0 = EmitScalarExpr(E->getArg(0)); 3282 Value *Arg1 = EmitScalarExpr(E->getArg(1)); 3283 3284 // crc32{c,}d intrinsics are implemnted as two calls to crc32{c,}w 3285 // intrinsics, hence we need different codegen for these cases. 3286 if (BuiltinID == ARM::BI__builtin_arm_crc32d || 3287 BuiltinID == ARM::BI__builtin_arm_crc32cd) { 3288 Value *C1 = llvm::ConstantInt::get(Int64Ty, 32); 3289 Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty); 3290 Value *Arg1b = Builder.CreateLShr(Arg1, C1); 3291 Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty); 3292 3293 Function *F = CGM.getIntrinsic(CRCIntrinsicID); 3294 Value *Res = Builder.CreateCall2(F, Arg0, Arg1a); 3295 return Builder.CreateCall2(F, Res, Arg1b); 3296 } else { 3297 Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty); 3298 3299 Function *F = CGM.getIntrinsic(CRCIntrinsicID); 3300 return Builder.CreateCall2(F, Arg0, Arg1); 3301 } 3302 } 3303 3304 SmallVector<Value*, 4> Ops; 3305 llvm::Value *Align = nullptr; 3306 for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) { 3307 if (i == 0) { 3308 switch (BuiltinID) { 3309 case NEON::BI__builtin_neon_vld1_v: 3310 case NEON::BI__builtin_neon_vld1q_v: 3311 case NEON::BI__builtin_neon_vld1q_lane_v: 3312 case NEON::BI__builtin_neon_vld1_lane_v: 3313 case NEON::BI__builtin_neon_vld1_dup_v: 3314 case NEON::BI__builtin_neon_vld1q_dup_v: 3315 case NEON::BI__builtin_neon_vst1_v: 3316 case NEON::BI__builtin_neon_vst1q_v: 3317 case NEON::BI__builtin_neon_vst1q_lane_v: 3318 case NEON::BI__builtin_neon_vst1_lane_v: 3319 case NEON::BI__builtin_neon_vst2_v: 3320 case NEON::BI__builtin_neon_vst2q_v: 3321 case NEON::BI__builtin_neon_vst2_lane_v: 3322 case NEON::BI__builtin_neon_vst2q_lane_v: 3323 case NEON::BI__builtin_neon_vst3_v: 3324 case NEON::BI__builtin_neon_vst3q_v: 3325 case NEON::BI__builtin_neon_vst3_lane_v: 3326 case NEON::BI__builtin_neon_vst3q_lane_v: 3327 case NEON::BI__builtin_neon_vst4_v: 3328 case NEON::BI__builtin_neon_vst4q_v: 3329 case NEON::BI__builtin_neon_vst4_lane_v: 3330 case NEON::BI__builtin_neon_vst4q_lane_v: 3331 // Get the alignment for the argument in addition to the value; 3332 // we'll use it later. 3333 std::pair<llvm::Value*, unsigned> Src = 3334 EmitPointerWithAlignment(E->getArg(0)); 3335 Ops.push_back(Src.first); 3336 Align = Builder.getInt32(Src.second); 3337 continue; 3338 } 3339 } 3340 if (i == 1) { 3341 switch (BuiltinID) { 3342 case NEON::BI__builtin_neon_vld2_v: 3343 case NEON::BI__builtin_neon_vld2q_v: 3344 case NEON::BI__builtin_neon_vld3_v: 3345 case NEON::BI__builtin_neon_vld3q_v: 3346 case NEON::BI__builtin_neon_vld4_v: 3347 case NEON::BI__builtin_neon_vld4q_v: 3348 case NEON::BI__builtin_neon_vld2_lane_v: 3349 case NEON::BI__builtin_neon_vld2q_lane_v: 3350 case NEON::BI__builtin_neon_vld3_lane_v: 3351 case NEON::BI__builtin_neon_vld3q_lane_v: 3352 case NEON::BI__builtin_neon_vld4_lane_v: 3353 case NEON::BI__builtin_neon_vld4q_lane_v: 3354 case NEON::BI__builtin_neon_vld2_dup_v: 3355 case NEON::BI__builtin_neon_vld3_dup_v: 3356 case NEON::BI__builtin_neon_vld4_dup_v: 3357 // Get the alignment for the argument in addition to the value; 3358 // we'll use it later. 3359 std::pair<llvm::Value*, unsigned> Src = 3360 EmitPointerWithAlignment(E->getArg(1)); 3361 Ops.push_back(Src.first); 3362 Align = Builder.getInt32(Src.second); 3363 continue; 3364 } 3365 } 3366 Ops.push_back(EmitScalarExpr(E->getArg(i))); 3367 } 3368 3369 switch (BuiltinID) { 3370 default: break; 3371 // vget_lane and vset_lane are not overloaded and do not have an extra 3372 // argument that specifies the vector type. 3373 case NEON::BI__builtin_neon_vget_lane_i8: 3374 case NEON::BI__builtin_neon_vget_lane_i16: 3375 case NEON::BI__builtin_neon_vget_lane_i32: 3376 case NEON::BI__builtin_neon_vget_lane_i64: 3377 case NEON::BI__builtin_neon_vget_lane_f32: 3378 case NEON::BI__builtin_neon_vgetq_lane_i8: 3379 case NEON::BI__builtin_neon_vgetq_lane_i16: 3380 case NEON::BI__builtin_neon_vgetq_lane_i32: 3381 case NEON::BI__builtin_neon_vgetq_lane_i64: 3382 case NEON::BI__builtin_neon_vgetq_lane_f32: 3383 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 3384 "vget_lane"); 3385 case NEON::BI__builtin_neon_vset_lane_i8: 3386 case NEON::BI__builtin_neon_vset_lane_i16: 3387 case NEON::BI__builtin_neon_vset_lane_i32: 3388 case NEON::BI__builtin_neon_vset_lane_i64: 3389 case NEON::BI__builtin_neon_vset_lane_f32: 3390 case NEON::BI__builtin_neon_vsetq_lane_i8: 3391 case NEON::BI__builtin_neon_vsetq_lane_i16: 3392 case NEON::BI__builtin_neon_vsetq_lane_i32: 3393 case NEON::BI__builtin_neon_vsetq_lane_i64: 3394 case NEON::BI__builtin_neon_vsetq_lane_f32: 3395 Ops.push_back(EmitScalarExpr(E->getArg(2))); 3396 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 3397 3398 // Non-polymorphic crypto instructions also not overloaded 3399 case NEON::BI__builtin_neon_vsha1h_u32: 3400 Ops.push_back(EmitScalarExpr(E->getArg(0))); 3401 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops, 3402 "vsha1h"); 3403 case NEON::BI__builtin_neon_vsha1cq_u32: 3404 Ops.push_back(EmitScalarExpr(E->getArg(2))); 3405 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops, 3406 "vsha1h"); 3407 case NEON::BI__builtin_neon_vsha1pq_u32: 3408 Ops.push_back(EmitScalarExpr(E->getArg(2))); 3409 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops, 3410 "vsha1h"); 3411 case NEON::BI__builtin_neon_vsha1mq_u32: 3412 Ops.push_back(EmitScalarExpr(E->getArg(2))); 3413 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops, 3414 "vsha1h"); 3415 } 3416 3417 // Get the last argument, which specifies the vector type. 3418 llvm::APSInt Result; 3419 const Expr *Arg = E->getArg(E->getNumArgs()-1); 3420 if (!Arg->isIntegerConstantExpr(Result, getContext())) 3421 return nullptr; 3422 3423 if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f || 3424 BuiltinID == ARM::BI__builtin_arm_vcvtr_d) { 3425 // Determine the overloaded type of this builtin. 3426 llvm::Type *Ty; 3427 if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f) 3428 Ty = FloatTy; 3429 else 3430 Ty = DoubleTy; 3431 3432 // Determine whether this is an unsigned conversion or not. 3433 bool usgn = Result.getZExtValue() == 1; 3434 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr; 3435 3436 // Call the appropriate intrinsic. 3437 Function *F = CGM.getIntrinsic(Int, Ty); 3438 return Builder.CreateCall(F, Ops, "vcvtr"); 3439 } 3440 3441 // Determine the type of this overloaded NEON intrinsic. 3442 NeonTypeFlags Type(Result.getZExtValue()); 3443 bool usgn = Type.isUnsigned(); 3444 bool rightShift = false; 3445 3446 llvm::VectorType *VTy = GetNeonType(this, Type); 3447 llvm::Type *Ty = VTy; 3448 if (!Ty) 3449 return nullptr; 3450 3451 // Many NEON builtins have identical semantics and uses in ARM and 3452 // AArch64. Emit these in a single function. 3453 auto IntrinsicMap = makeArrayRef(ARMSIMDIntrinsicMap); 3454 const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap( 3455 IntrinsicMap, BuiltinID, NEONSIMDIntrinsicsProvenSorted); 3456 if (Builtin) 3457 return EmitCommonNeonBuiltinExpr( 3458 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic, 3459 Builtin->NameHint, Builtin->TypeModifier, E, Ops, Align); 3460 3461 unsigned Int; 3462 switch (BuiltinID) { 3463 default: return nullptr; 3464 case NEON::BI__builtin_neon_vld1q_lane_v: 3465 // Handle 64-bit integer elements as a special case. Use shuffles of 3466 // one-element vectors to avoid poor code for i64 in the backend. 3467 if (VTy->getElementType()->isIntegerTy(64)) { 3468 // Extract the other lane. 3469 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3470 int Lane = cast<ConstantInt>(Ops[2])->getZExtValue(); 3471 Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane)); 3472 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); 3473 // Load the value as a one-element vector. 3474 Ty = llvm::VectorType::get(VTy->getElementType(), 1); 3475 Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Ty); 3476 Value *Ld = Builder.CreateCall2(F, Ops[0], Align); 3477 // Combine them. 3478 SmallVector<Constant*, 2> Indices; 3479 Indices.push_back(ConstantInt::get(Int32Ty, 1-Lane)); 3480 Indices.push_back(ConstantInt::get(Int32Ty, Lane)); 3481 SV = llvm::ConstantVector::get(Indices); 3482 return Builder.CreateShuffleVector(Ops[1], Ld, SV, "vld1q_lane"); 3483 } 3484 // fall through 3485 case NEON::BI__builtin_neon_vld1_lane_v: { 3486 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3487 Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 3488 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3489 LoadInst *Ld = Builder.CreateLoad(Ops[0]); 3490 Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue()); 3491 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane"); 3492 } 3493 case NEON::BI__builtin_neon_vld2_dup_v: 3494 case NEON::BI__builtin_neon_vld3_dup_v: 3495 case NEON::BI__builtin_neon_vld4_dup_v: { 3496 // Handle 64-bit elements as a special-case. There is no "dup" needed. 3497 if (VTy->getElementType()->getPrimitiveSizeInBits() == 64) { 3498 switch (BuiltinID) { 3499 case NEON::BI__builtin_neon_vld2_dup_v: 3500 Int = Intrinsic::arm_neon_vld2; 3501 break; 3502 case NEON::BI__builtin_neon_vld3_dup_v: 3503 Int = Intrinsic::arm_neon_vld3; 3504 break; 3505 case NEON::BI__builtin_neon_vld4_dup_v: 3506 Int = Intrinsic::arm_neon_vld4; 3507 break; 3508 default: llvm_unreachable("unknown vld_dup intrinsic?"); 3509 } 3510 Function *F = CGM.getIntrinsic(Int, Ty); 3511 Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld_dup"); 3512 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 3513 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3514 return Builder.CreateStore(Ops[1], Ops[0]); 3515 } 3516 switch (BuiltinID) { 3517 case NEON::BI__builtin_neon_vld2_dup_v: 3518 Int = Intrinsic::arm_neon_vld2lane; 3519 break; 3520 case NEON::BI__builtin_neon_vld3_dup_v: 3521 Int = Intrinsic::arm_neon_vld3lane; 3522 break; 3523 case NEON::BI__builtin_neon_vld4_dup_v: 3524 Int = Intrinsic::arm_neon_vld4lane; 3525 break; 3526 default: llvm_unreachable("unknown vld_dup intrinsic?"); 3527 } 3528 Function *F = CGM.getIntrinsic(Int, Ty); 3529 llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType()); 3530 3531 SmallVector<Value*, 6> Args; 3532 Args.push_back(Ops[1]); 3533 Args.append(STy->getNumElements(), UndefValue::get(Ty)); 3534 3535 llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); 3536 Args.push_back(CI); 3537 Args.push_back(Align); 3538 3539 Ops[1] = Builder.CreateCall(F, Args, "vld_dup"); 3540 // splat lane 0 to all elts in each vector of the result. 3541 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { 3542 Value *Val = Builder.CreateExtractValue(Ops[1], i); 3543 Value *Elt = Builder.CreateBitCast(Val, Ty); 3544 Elt = EmitNeonSplat(Elt, CI); 3545 Elt = Builder.CreateBitCast(Elt, Val->getType()); 3546 Ops[1] = Builder.CreateInsertValue(Ops[1], Elt, i); 3547 } 3548 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 3549 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3550 return Builder.CreateStore(Ops[1], Ops[0]); 3551 } 3552 case NEON::BI__builtin_neon_vqrshrn_n_v: 3553 Int = 3554 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns; 3555 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n", 3556 1, true); 3557 case NEON::BI__builtin_neon_vqrshrun_n_v: 3558 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty), 3559 Ops, "vqrshrun_n", 1, true); 3560 case NEON::BI__builtin_neon_vqshrn_n_v: 3561 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns; 3562 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n", 3563 1, true); 3564 case NEON::BI__builtin_neon_vqshrun_n_v: 3565 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty), 3566 Ops, "vqshrun_n", 1, true); 3567 case NEON::BI__builtin_neon_vrecpe_v: 3568 case NEON::BI__builtin_neon_vrecpeq_v: 3569 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty), 3570 Ops, "vrecpe"); 3571 case NEON::BI__builtin_neon_vrshrn_n_v: 3572 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty), 3573 Ops, "vrshrn_n", 1, true); 3574 case NEON::BI__builtin_neon_vrsra_n_v: 3575 case NEON::BI__builtin_neon_vrsraq_n_v: 3576 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3577 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3578 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true); 3579 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts; 3580 Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Ty), Ops[1], Ops[2]); 3581 return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n"); 3582 case NEON::BI__builtin_neon_vsri_n_v: 3583 case NEON::BI__builtin_neon_vsriq_n_v: 3584 rightShift = true; 3585 case NEON::BI__builtin_neon_vsli_n_v: 3586 case NEON::BI__builtin_neon_vsliq_n_v: 3587 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift); 3588 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty), 3589 Ops, "vsli_n"); 3590 case NEON::BI__builtin_neon_vsra_n_v: 3591 case NEON::BI__builtin_neon_vsraq_n_v: 3592 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3593 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); 3594 return Builder.CreateAdd(Ops[0], Ops[1]); 3595 case NEON::BI__builtin_neon_vst1q_lane_v: 3596 // Handle 64-bit integer elements as a special case. Use a shuffle to get 3597 // a one-element vector and avoid poor code for i64 in the backend. 3598 if (VTy->getElementType()->isIntegerTy(64)) { 3599 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3600 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2])); 3601 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); 3602 Ops[2] = Align; 3603 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1, 3604 Ops[1]->getType()), Ops); 3605 } 3606 // fall through 3607 case NEON::BI__builtin_neon_vst1_lane_v: { 3608 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3609 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); 3610 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 3611 StoreInst *St = Builder.CreateStore(Ops[1], 3612 Builder.CreateBitCast(Ops[0], Ty)); 3613 St->setAlignment(cast<ConstantInt>(Align)->getZExtValue()); 3614 return St; 3615 } 3616 case NEON::BI__builtin_neon_vtbl1_v: 3617 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1), 3618 Ops, "vtbl1"); 3619 case NEON::BI__builtin_neon_vtbl2_v: 3620 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2), 3621 Ops, "vtbl2"); 3622 case NEON::BI__builtin_neon_vtbl3_v: 3623 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3), 3624 Ops, "vtbl3"); 3625 case NEON::BI__builtin_neon_vtbl4_v: 3626 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4), 3627 Ops, "vtbl4"); 3628 case NEON::BI__builtin_neon_vtbx1_v: 3629 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1), 3630 Ops, "vtbx1"); 3631 case NEON::BI__builtin_neon_vtbx2_v: 3632 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2), 3633 Ops, "vtbx2"); 3634 case NEON::BI__builtin_neon_vtbx3_v: 3635 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3), 3636 Ops, "vtbx3"); 3637 case NEON::BI__builtin_neon_vtbx4_v: 3638 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4), 3639 Ops, "vtbx4"); 3640 } 3641 } 3642 3643 static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, 3644 const CallExpr *E, 3645 SmallVectorImpl<Value *> &Ops) { 3646 unsigned int Int = 0; 3647 const char *s = nullptr; 3648 3649 switch (BuiltinID) { 3650 default: 3651 return nullptr; 3652 case NEON::BI__builtin_neon_vtbl1_v: 3653 case NEON::BI__builtin_neon_vqtbl1_v: 3654 case NEON::BI__builtin_neon_vqtbl1q_v: 3655 case NEON::BI__builtin_neon_vtbl2_v: 3656 case NEON::BI__builtin_neon_vqtbl2_v: 3657 case NEON::BI__builtin_neon_vqtbl2q_v: 3658 case NEON::BI__builtin_neon_vtbl3_v: 3659 case NEON::BI__builtin_neon_vqtbl3_v: 3660 case NEON::BI__builtin_neon_vqtbl3q_v: 3661 case NEON::BI__builtin_neon_vtbl4_v: 3662 case NEON::BI__builtin_neon_vqtbl4_v: 3663 case NEON::BI__builtin_neon_vqtbl4q_v: 3664 break; 3665 case NEON::BI__builtin_neon_vtbx1_v: 3666 case NEON::BI__builtin_neon_vqtbx1_v: 3667 case NEON::BI__builtin_neon_vqtbx1q_v: 3668 case NEON::BI__builtin_neon_vtbx2_v: 3669 case NEON::BI__builtin_neon_vqtbx2_v: 3670 case NEON::BI__builtin_neon_vqtbx2q_v: 3671 case NEON::BI__builtin_neon_vtbx3_v: 3672 case NEON::BI__builtin_neon_vqtbx3_v: 3673 case NEON::BI__builtin_neon_vqtbx3q_v: 3674 case NEON::BI__builtin_neon_vtbx4_v: 3675 case NEON::BI__builtin_neon_vqtbx4_v: 3676 case NEON::BI__builtin_neon_vqtbx4q_v: 3677 break; 3678 } 3679 3680 assert(E->getNumArgs() >= 3); 3681 3682 // Get the last argument, which specifies the vector type. 3683 llvm::APSInt Result; 3684 const Expr *Arg = E->getArg(E->getNumArgs() - 1); 3685 if (!Arg->isIntegerConstantExpr(Result, CGF.getContext())) 3686 return nullptr; 3687 3688 // Determine the type of this overloaded NEON intrinsic. 3689 NeonTypeFlags Type(Result.getZExtValue()); 3690 llvm::VectorType *VTy = GetNeonType(&CGF, Type); 3691 llvm::Type *Ty = VTy; 3692 if (!Ty) 3693 return nullptr; 3694 3695 unsigned nElts = VTy->getNumElements(); 3696 3697 CodeGen::CGBuilderTy &Builder = CGF.Builder; 3698 3699 // AArch64 scalar builtins are not overloaded, they do not have an extra 3700 // argument that specifies the vector type, need to handle each case. 3701 SmallVector<Value *, 2> TblOps; 3702 switch (BuiltinID) { 3703 case NEON::BI__builtin_neon_vtbl1_v: { 3704 TblOps.push_back(Ops[0]); 3705 return packTBLDVectorList(CGF, TblOps, nullptr, Ops[1], Ty, 3706 Intrinsic::aarch64_neon_tbl1, "vtbl1"); 3707 } 3708 case NEON::BI__builtin_neon_vtbl2_v: { 3709 TblOps.push_back(Ops[0]); 3710 TblOps.push_back(Ops[1]); 3711 return packTBLDVectorList(CGF, TblOps, nullptr, Ops[2], Ty, 3712 Intrinsic::aarch64_neon_tbl1, "vtbl1"); 3713 } 3714 case NEON::BI__builtin_neon_vtbl3_v: { 3715 TblOps.push_back(Ops[0]); 3716 TblOps.push_back(Ops[1]); 3717 TblOps.push_back(Ops[2]); 3718 return packTBLDVectorList(CGF, TblOps, nullptr, Ops[3], Ty, 3719 Intrinsic::aarch64_neon_tbl2, "vtbl2"); 3720 } 3721 case NEON::BI__builtin_neon_vtbl4_v: { 3722 TblOps.push_back(Ops[0]); 3723 TblOps.push_back(Ops[1]); 3724 TblOps.push_back(Ops[2]); 3725 TblOps.push_back(Ops[3]); 3726 return packTBLDVectorList(CGF, TblOps, nullptr, Ops[4], Ty, 3727 Intrinsic::aarch64_neon_tbl2, "vtbl2"); 3728 } 3729 case NEON::BI__builtin_neon_vtbx1_v: { 3730 TblOps.push_back(Ops[1]); 3731 Value *TblRes = packTBLDVectorList(CGF, TblOps, nullptr, Ops[2], Ty, 3732 Intrinsic::aarch64_neon_tbl1, "vtbl1"); 3733 3734 llvm::Constant *Eight = ConstantInt::get(VTy->getElementType(), 8); 3735 Value* EightV = llvm::ConstantVector::getSplat(nElts, Eight); 3736 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV); 3737 CmpRes = Builder.CreateSExt(CmpRes, Ty); 3738 3739 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); 3740 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes); 3741 return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx"); 3742 } 3743 case NEON::BI__builtin_neon_vtbx2_v: { 3744 TblOps.push_back(Ops[1]); 3745 TblOps.push_back(Ops[2]); 3746 return packTBLDVectorList(CGF, TblOps, Ops[0], Ops[3], Ty, 3747 Intrinsic::aarch64_neon_tbx1, "vtbx1"); 3748 } 3749 case NEON::BI__builtin_neon_vtbx3_v: { 3750 TblOps.push_back(Ops[1]); 3751 TblOps.push_back(Ops[2]); 3752 TblOps.push_back(Ops[3]); 3753 Value *TblRes = packTBLDVectorList(CGF, TblOps, nullptr, Ops[4], Ty, 3754 Intrinsic::aarch64_neon_tbl2, "vtbl2"); 3755 3756 llvm::Constant *TwentyFour = ConstantInt::get(VTy->getElementType(), 24); 3757 Value* TwentyFourV = llvm::ConstantVector::getSplat(nElts, TwentyFour); 3758 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4], 3759 TwentyFourV); 3760 CmpRes = Builder.CreateSExt(CmpRes, Ty); 3761 3762 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); 3763 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes); 3764 return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx"); 3765 } 3766 case NEON::BI__builtin_neon_vtbx4_v: { 3767 TblOps.push_back(Ops[1]); 3768 TblOps.push_back(Ops[2]); 3769 TblOps.push_back(Ops[3]); 3770 TblOps.push_back(Ops[4]); 3771 return packTBLDVectorList(CGF, TblOps, Ops[0], Ops[5], Ty, 3772 Intrinsic::aarch64_neon_tbx2, "vtbx2"); 3773 } 3774 case NEON::BI__builtin_neon_vqtbl1_v: 3775 case NEON::BI__builtin_neon_vqtbl1q_v: 3776 Int = Intrinsic::aarch64_neon_tbl1; s = "vtbl1"; break; 3777 case NEON::BI__builtin_neon_vqtbl2_v: 3778 case NEON::BI__builtin_neon_vqtbl2q_v: { 3779 Int = Intrinsic::aarch64_neon_tbl2; s = "vtbl2"; break; 3780 case NEON::BI__builtin_neon_vqtbl3_v: 3781 case NEON::BI__builtin_neon_vqtbl3q_v: 3782 Int = Intrinsic::aarch64_neon_tbl3; s = "vtbl3"; break; 3783 case NEON::BI__builtin_neon_vqtbl4_v: 3784 case NEON::BI__builtin_neon_vqtbl4q_v: 3785 Int = Intrinsic::aarch64_neon_tbl4; s = "vtbl4"; break; 3786 case NEON::BI__builtin_neon_vqtbx1_v: 3787 case NEON::BI__builtin_neon_vqtbx1q_v: 3788 Int = Intrinsic::aarch64_neon_tbx1; s = "vtbx1"; break; 3789 case NEON::BI__builtin_neon_vqtbx2_v: 3790 case NEON::BI__builtin_neon_vqtbx2q_v: 3791 Int = Intrinsic::aarch64_neon_tbx2; s = "vtbx2"; break; 3792 case NEON::BI__builtin_neon_vqtbx3_v: 3793 case NEON::BI__builtin_neon_vqtbx3q_v: 3794 Int = Intrinsic::aarch64_neon_tbx3; s = "vtbx3"; break; 3795 case NEON::BI__builtin_neon_vqtbx4_v: 3796 case NEON::BI__builtin_neon_vqtbx4q_v: 3797 Int = Intrinsic::aarch64_neon_tbx4; s = "vtbx4"; break; 3798 } 3799 } 3800 3801 if (!Int) 3802 return nullptr; 3803 3804 Function *F = CGF.CGM.getIntrinsic(Int, Ty); 3805 return CGF.EmitNeonCall(F, Ops, s); 3806 } 3807 3808 Value *CodeGenFunction::vectorWrapScalar16(Value *Op) { 3809 llvm::Type *VTy = llvm::VectorType::get(Int16Ty, 4); 3810 Op = Builder.CreateBitCast(Op, Int16Ty); 3811 Value *V = UndefValue::get(VTy); 3812 llvm::Constant *CI = ConstantInt::get(SizeTy, 0); 3813 Op = Builder.CreateInsertElement(V, Op, CI); 3814 return Op; 3815 } 3816 3817 Value *CodeGenFunction::vectorWrapScalar8(Value *Op) { 3818 llvm::Type *VTy = llvm::VectorType::get(Int8Ty, 8); 3819 Op = Builder.CreateBitCast(Op, Int8Ty); 3820 Value *V = UndefValue::get(VTy); 3821 llvm::Constant *CI = ConstantInt::get(SizeTy, 0); 3822 Op = Builder.CreateInsertElement(V, Op, CI); 3823 return Op; 3824 } 3825 3826 Value *CodeGenFunction:: 3827 emitVectorWrappedScalar8Intrinsic(unsigned Int, SmallVectorImpl<Value*> &Ops, 3828 const char *Name) { 3829 // i8 is not a legal types for AArch64, so we can't just use 3830 // a normal overloaded intrinsic call for these scalar types. Instead 3831 // we'll build 64-bit vectors w/ lane zero being our input values and 3832 // perform the operation on that. The back end can pattern match directly 3833 // to the scalar instruction. 3834 Ops[0] = vectorWrapScalar8(Ops[0]); 3835 Ops[1] = vectorWrapScalar8(Ops[1]); 3836 llvm::Type *VTy = llvm::VectorType::get(Int8Ty, 8); 3837 Value *V = EmitNeonCall(CGM.getIntrinsic(Int, VTy), Ops, Name); 3838 Constant *CI = ConstantInt::get(SizeTy, 0); 3839 return Builder.CreateExtractElement(V, CI, "lane0"); 3840 } 3841 3842 Value *CodeGenFunction:: 3843 emitVectorWrappedScalar16Intrinsic(unsigned Int, SmallVectorImpl<Value*> &Ops, 3844 const char *Name) { 3845 // i16 is not a legal types for AArch64, so we can't just use 3846 // a normal overloaded intrinsic call for these scalar types. Instead 3847 // we'll build 64-bit vectors w/ lane zero being our input values and 3848 // perform the operation on that. The back end can pattern match directly 3849 // to the scalar instruction. 3850 Ops[0] = vectorWrapScalar16(Ops[0]); 3851 Ops[1] = vectorWrapScalar16(Ops[1]); 3852 llvm::Type *VTy = llvm::VectorType::get(Int16Ty, 4); 3853 Value *V = EmitNeonCall(CGM.getIntrinsic(Int, VTy), Ops, Name); 3854 Constant *CI = ConstantInt::get(SizeTy, 0); 3855 return Builder.CreateExtractElement(V, CI, "lane0"); 3856 } 3857 3858 Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, 3859 const CallExpr *E) { 3860 unsigned HintID = static_cast<unsigned>(-1); 3861 switch (BuiltinID) { 3862 default: break; 3863 case AArch64::BI__builtin_arm_nop: 3864 HintID = 0; 3865 break; 3866 case AArch64::BI__builtin_arm_yield: 3867 HintID = 1; 3868 break; 3869 case AArch64::BI__builtin_arm_wfe: 3870 HintID = 2; 3871 break; 3872 case AArch64::BI__builtin_arm_wfi: 3873 HintID = 3; 3874 break; 3875 case AArch64::BI__builtin_arm_sev: 3876 HintID = 4; 3877 break; 3878 case AArch64::BI__builtin_arm_sevl: 3879 HintID = 5; 3880 break; 3881 } 3882 3883 if (HintID != static_cast<unsigned>(-1)) { 3884 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hint); 3885 return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID)); 3886 } 3887 3888 if (BuiltinID == AArch64::BI__builtin_arm_prefetch) { 3889 Value *Address = EmitScalarExpr(E->getArg(0)); 3890 Value *RW = EmitScalarExpr(E->getArg(1)); 3891 Value *CacheLevel = EmitScalarExpr(E->getArg(2)); 3892 Value *RetentionPolicy = EmitScalarExpr(E->getArg(3)); 3893 Value *IsData = EmitScalarExpr(E->getArg(4)); 3894 3895 Value *Locality = nullptr; 3896 if (cast<llvm::ConstantInt>(RetentionPolicy)->isZero()) { 3897 // Temporal fetch, needs to convert cache level to locality. 3898 Locality = llvm::ConstantInt::get(Int32Ty, 3899 -cast<llvm::ConstantInt>(CacheLevel)->getValue() + 3); 3900 } else { 3901 // Streaming fetch. 3902 Locality = llvm::ConstantInt::get(Int32Ty, 0); 3903 } 3904 3905 // FIXME: We need AArch64 specific LLVM intrinsic if we want to specify 3906 // PLDL3STRM or PLDL2STRM. 3907 Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 3908 return Builder.CreateCall4(F, Address, RW, Locality, IsData); 3909 } 3910 3911 if (BuiltinID == AArch64::BI__builtin_arm_rbit) { 3912 assert((getContext().getTypeSize(E->getType()) == 32) && 3913 "rbit of unusual size!"); 3914 llvm::Value *Arg = EmitScalarExpr(E->getArg(0)); 3915 return Builder.CreateCall( 3916 CGM.getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg, "rbit"); 3917 } 3918 if (BuiltinID == AArch64::BI__builtin_arm_rbit64) { 3919 assert((getContext().getTypeSize(E->getType()) == 64) && 3920 "rbit of unusual size!"); 3921 llvm::Value *Arg = EmitScalarExpr(E->getArg(0)); 3922 return Builder.CreateCall( 3923 CGM.getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg, "rbit"); 3924 } 3925 3926 if (BuiltinID == AArch64::BI__clear_cache) { 3927 assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments"); 3928 const FunctionDecl *FD = E->getDirectCallee(); 3929 SmallVector<Value*, 2> Ops; 3930 for (unsigned i = 0; i < 2; i++) 3931 Ops.push_back(EmitScalarExpr(E->getArg(i))); 3932 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType()); 3933 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty); 3934 StringRef Name = FD->getName(); 3935 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); 3936 } 3937 3938 if ((BuiltinID == AArch64::BI__builtin_arm_ldrex || 3939 BuiltinID == AArch64::BI__builtin_arm_ldaex) && 3940 getContext().getTypeSize(E->getType()) == 128) { 3941 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex 3942 ? Intrinsic::aarch64_ldaxp 3943 : Intrinsic::aarch64_ldxp); 3944 3945 Value *LdPtr = EmitScalarExpr(E->getArg(0)); 3946 Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy), 3947 "ldxp"); 3948 3949 Value *Val0 = Builder.CreateExtractValue(Val, 1); 3950 Value *Val1 = Builder.CreateExtractValue(Val, 0); 3951 llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128); 3952 Val0 = Builder.CreateZExt(Val0, Int128Ty); 3953 Val1 = Builder.CreateZExt(Val1, Int128Ty); 3954 3955 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64); 3956 Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */); 3957 Val = Builder.CreateOr(Val, Val1); 3958 return Builder.CreateBitCast(Val, ConvertType(E->getType())); 3959 } else if (BuiltinID == AArch64::BI__builtin_arm_ldrex || 3960 BuiltinID == AArch64::BI__builtin_arm_ldaex) { 3961 Value *LoadAddr = EmitScalarExpr(E->getArg(0)); 3962 3963 QualType Ty = E->getType(); 3964 llvm::Type *RealResTy = ConvertType(Ty); 3965 llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(), 3966 getContext().getTypeSize(Ty)); 3967 LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo()); 3968 3969 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex 3970 ? Intrinsic::aarch64_ldaxr 3971 : Intrinsic::aarch64_ldxr, 3972 LoadAddr->getType()); 3973 Value *Val = Builder.CreateCall(F, LoadAddr, "ldxr"); 3974 3975 if (RealResTy->isPointerTy()) 3976 return Builder.CreateIntToPtr(Val, RealResTy); 3977 3978 Val = Builder.CreateTruncOrBitCast(Val, IntResTy); 3979 return Builder.CreateBitCast(Val, RealResTy); 3980 } 3981 3982 if ((BuiltinID == AArch64::BI__builtin_arm_strex || 3983 BuiltinID == AArch64::BI__builtin_arm_stlex) && 3984 getContext().getTypeSize(E->getArg(0)->getType()) == 128) { 3985 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex 3986 ? Intrinsic::aarch64_stlxp 3987 : Intrinsic::aarch64_stxp); 3988 llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty, NULL); 3989 3990 Value *One = llvm::ConstantInt::get(Int32Ty, 1); 3991 Value *Tmp = Builder.CreateAlloca(ConvertType(E->getArg(0)->getType()), 3992 One); 3993 Value *Val = EmitScalarExpr(E->getArg(0)); 3994 Builder.CreateStore(Val, Tmp); 3995 3996 Value *LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy)); 3997 Val = Builder.CreateLoad(LdPtr); 3998 3999 Value *Arg0 = Builder.CreateExtractValue(Val, 0); 4000 Value *Arg1 = Builder.CreateExtractValue(Val, 1); 4001 Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), 4002 Int8PtrTy); 4003 return Builder.CreateCall3(F, Arg0, Arg1, StPtr, "stxp"); 4004 } else if (BuiltinID == AArch64::BI__builtin_arm_strex || 4005 BuiltinID == AArch64::BI__builtin_arm_stlex) { 4006 Value *StoreVal = EmitScalarExpr(E->getArg(0)); 4007 Value *StoreAddr = EmitScalarExpr(E->getArg(1)); 4008 4009 QualType Ty = E->getArg(0)->getType(); 4010 llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(), 4011 getContext().getTypeSize(Ty)); 4012 StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo()); 4013 4014 if (StoreVal->getType()->isPointerTy()) 4015 StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty); 4016 else { 4017 StoreVal = Builder.CreateBitCast(StoreVal, StoreTy); 4018 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty); 4019 } 4020 4021 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex 4022 ? Intrinsic::aarch64_stlxr 4023 : Intrinsic::aarch64_stxr, 4024 StoreAddr->getType()); 4025 return Builder.CreateCall2(F, StoreVal, StoreAddr, "stxr"); 4026 } 4027 4028 if (BuiltinID == AArch64::BI__builtin_arm_clrex) { 4029 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex); 4030 return Builder.CreateCall(F); 4031 } 4032 4033 // CRC32 4034 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic; 4035 switch (BuiltinID) { 4036 case AArch64::BI__builtin_arm_crc32b: 4037 CRCIntrinsicID = Intrinsic::aarch64_crc32b; break; 4038 case AArch64::BI__builtin_arm_crc32cb: 4039 CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break; 4040 case AArch64::BI__builtin_arm_crc32h: 4041 CRCIntrinsicID = Intrinsic::aarch64_crc32h; break; 4042 case AArch64::BI__builtin_arm_crc32ch: 4043 CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break; 4044 case AArch64::BI__builtin_arm_crc32w: 4045 CRCIntrinsicID = Intrinsic::aarch64_crc32w; break; 4046 case AArch64::BI__builtin_arm_crc32cw: 4047 CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break; 4048 case AArch64::BI__builtin_arm_crc32d: 4049 CRCIntrinsicID = Intrinsic::aarch64_crc32x; break; 4050 case AArch64::BI__builtin_arm_crc32cd: 4051 CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break; 4052 } 4053 4054 if (CRCIntrinsicID != Intrinsic::not_intrinsic) { 4055 Value *Arg0 = EmitScalarExpr(E->getArg(0)); 4056 Value *Arg1 = EmitScalarExpr(E->getArg(1)); 4057 Function *F = CGM.getIntrinsic(CRCIntrinsicID); 4058 4059 llvm::Type *DataTy = F->getFunctionType()->getParamType(1); 4060 Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy); 4061 4062 return Builder.CreateCall2(F, Arg0, Arg1); 4063 } 4064 4065 llvm::SmallVector<Value*, 4> Ops; 4066 for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) 4067 Ops.push_back(EmitScalarExpr(E->getArg(i))); 4068 4069 auto SISDMap = makeArrayRef(AArch64SISDIntrinsicMap); 4070 const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap( 4071 SISDMap, BuiltinID, AArch64SISDIntrinsicsProvenSorted); 4072 4073 if (Builtin) { 4074 Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1))); 4075 Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E); 4076 assert(Result && "SISD intrinsic should have been handled"); 4077 return Result; 4078 } 4079 4080 llvm::APSInt Result; 4081 const Expr *Arg = E->getArg(E->getNumArgs()-1); 4082 NeonTypeFlags Type(0); 4083 if (Arg->isIntegerConstantExpr(Result, getContext())) 4084 // Determine the type of this overloaded NEON intrinsic. 4085 Type = NeonTypeFlags(Result.getZExtValue()); 4086 4087 bool usgn = Type.isUnsigned(); 4088 bool quad = Type.isQuad(); 4089 4090 // Handle non-overloaded intrinsics first. 4091 switch (BuiltinID) { 4092 default: break; 4093 case NEON::BI__builtin_neon_vldrq_p128: { 4094 llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128); 4095 Value *Ptr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int128PTy); 4096 return Builder.CreateLoad(Ptr); 4097 } 4098 case NEON::BI__builtin_neon_vstrq_p128: { 4099 llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128); 4100 Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy); 4101 return Builder.CreateStore(EmitScalarExpr(E->getArg(1)), Ptr); 4102 } 4103 case NEON::BI__builtin_neon_vcvts_u32_f32: 4104 case NEON::BI__builtin_neon_vcvtd_u64_f64: 4105 usgn = true; 4106 // FALL THROUGH 4107 case NEON::BI__builtin_neon_vcvts_s32_f32: 4108 case NEON::BI__builtin_neon_vcvtd_s64_f64: { 4109 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4110 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; 4111 llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty; 4112 llvm::Type *FTy = Is64 ? DoubleTy : FloatTy; 4113 Ops[0] = Builder.CreateBitCast(Ops[0], FTy); 4114 if (usgn) 4115 return Builder.CreateFPToUI(Ops[0], InTy); 4116 return Builder.CreateFPToSI(Ops[0], InTy); 4117 } 4118 case NEON::BI__builtin_neon_vcvts_f32_u32: 4119 case NEON::BI__builtin_neon_vcvtd_f64_u64: 4120 usgn = true; 4121 // FALL THROUGH 4122 case NEON::BI__builtin_neon_vcvts_f32_s32: 4123 case NEON::BI__builtin_neon_vcvtd_f64_s64: { 4124 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4125 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; 4126 llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty; 4127 llvm::Type *FTy = Is64 ? DoubleTy : FloatTy; 4128 Ops[0] = Builder.CreateBitCast(Ops[0], InTy); 4129 if (usgn) 4130 return Builder.CreateUIToFP(Ops[0], FTy); 4131 return Builder.CreateSIToFP(Ops[0], FTy); 4132 } 4133 case NEON::BI__builtin_neon_vpaddd_s64: { 4134 llvm::Type *Ty = 4135 llvm::VectorType::get(llvm::Type::getInt64Ty(getLLVMContext()), 2); 4136 Value *Vec = EmitScalarExpr(E->getArg(0)); 4137 // The vector is v2f64, so make sure it's bitcast to that. 4138 Vec = Builder.CreateBitCast(Vec, Ty, "v2i64"); 4139 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); 4140 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); 4141 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 4142 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 4143 // Pairwise addition of a v2f64 into a scalar f64. 4144 return Builder.CreateAdd(Op0, Op1, "vpaddd"); 4145 } 4146 case NEON::BI__builtin_neon_vpaddd_f64: { 4147 llvm::Type *Ty = 4148 llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2); 4149 Value *Vec = EmitScalarExpr(E->getArg(0)); 4150 // The vector is v2f64, so make sure it's bitcast to that. 4151 Vec = Builder.CreateBitCast(Vec, Ty, "v2f64"); 4152 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); 4153 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); 4154 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 4155 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 4156 // Pairwise addition of a v2f64 into a scalar f64. 4157 return Builder.CreateFAdd(Op0, Op1, "vpaddd"); 4158 } 4159 case NEON::BI__builtin_neon_vpadds_f32: { 4160 llvm::Type *Ty = 4161 llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2); 4162 Value *Vec = EmitScalarExpr(E->getArg(0)); 4163 // The vector is v2f32, so make sure it's bitcast to that. 4164 Vec = Builder.CreateBitCast(Vec, Ty, "v2f32"); 4165 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); 4166 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); 4167 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 4168 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 4169 // Pairwise addition of a v2f32 into a scalar f32. 4170 return Builder.CreateFAdd(Op0, Op1, "vpaddd"); 4171 } 4172 case NEON::BI__builtin_neon_vceqzd_s64: 4173 case NEON::BI__builtin_neon_vceqzd_f64: 4174 case NEON::BI__builtin_neon_vceqzs_f32: 4175 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4176 return EmitAArch64CompareBuiltinExpr( 4177 Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OEQ, 4178 ICmpInst::ICMP_EQ, "vceqz"); 4179 case NEON::BI__builtin_neon_vcgezd_s64: 4180 case NEON::BI__builtin_neon_vcgezd_f64: 4181 case NEON::BI__builtin_neon_vcgezs_f32: 4182 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4183 return EmitAArch64CompareBuiltinExpr( 4184 Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OGE, 4185 ICmpInst::ICMP_SGE, "vcgez"); 4186 case NEON::BI__builtin_neon_vclezd_s64: 4187 case NEON::BI__builtin_neon_vclezd_f64: 4188 case NEON::BI__builtin_neon_vclezs_f32: 4189 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4190 return EmitAArch64CompareBuiltinExpr( 4191 Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OLE, 4192 ICmpInst::ICMP_SLE, "vclez"); 4193 case NEON::BI__builtin_neon_vcgtzd_s64: 4194 case NEON::BI__builtin_neon_vcgtzd_f64: 4195 case NEON::BI__builtin_neon_vcgtzs_f32: 4196 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4197 return EmitAArch64CompareBuiltinExpr( 4198 Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OGT, 4199 ICmpInst::ICMP_SGT, "vcgtz"); 4200 case NEON::BI__builtin_neon_vcltzd_s64: 4201 case NEON::BI__builtin_neon_vcltzd_f64: 4202 case NEON::BI__builtin_neon_vcltzs_f32: 4203 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4204 return EmitAArch64CompareBuiltinExpr( 4205 Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OLT, 4206 ICmpInst::ICMP_SLT, "vcltz"); 4207 4208 case NEON::BI__builtin_neon_vceqzd_u64: { 4209 llvm::Type *Ty = llvm::Type::getInt64Ty(getLLVMContext()); 4210 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4211 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4212 Ops[0] = Builder.CreateICmp(llvm::ICmpInst::ICMP_EQ, Ops[0], 4213 llvm::Constant::getNullValue(Ty)); 4214 return Builder.CreateSExt(Ops[0], Ty, "vceqzd"); 4215 } 4216 case NEON::BI__builtin_neon_vceqd_f64: 4217 case NEON::BI__builtin_neon_vcled_f64: 4218 case NEON::BI__builtin_neon_vcltd_f64: 4219 case NEON::BI__builtin_neon_vcged_f64: 4220 case NEON::BI__builtin_neon_vcgtd_f64: { 4221 llvm::CmpInst::Predicate P; 4222 switch (BuiltinID) { 4223 default: llvm_unreachable("missing builtin ID in switch!"); 4224 case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ; break; 4225 case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE; break; 4226 case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT; break; 4227 case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE; break; 4228 case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT; break; 4229 } 4230 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4231 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 4232 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); 4233 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); 4234 return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd"); 4235 } 4236 case NEON::BI__builtin_neon_vceqs_f32: 4237 case NEON::BI__builtin_neon_vcles_f32: 4238 case NEON::BI__builtin_neon_vclts_f32: 4239 case NEON::BI__builtin_neon_vcges_f32: 4240 case NEON::BI__builtin_neon_vcgts_f32: { 4241 llvm::CmpInst::Predicate P; 4242 switch (BuiltinID) { 4243 default: llvm_unreachable("missing builtin ID in switch!"); 4244 case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ; break; 4245 case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE; break; 4246 case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT; break; 4247 case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE; break; 4248 case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT; break; 4249 } 4250 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4251 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy); 4252 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy); 4253 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); 4254 return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd"); 4255 } 4256 case NEON::BI__builtin_neon_vceqd_s64: 4257 case NEON::BI__builtin_neon_vceqd_u64: 4258 case NEON::BI__builtin_neon_vcgtd_s64: 4259 case NEON::BI__builtin_neon_vcgtd_u64: 4260 case NEON::BI__builtin_neon_vcltd_s64: 4261 case NEON::BI__builtin_neon_vcltd_u64: 4262 case NEON::BI__builtin_neon_vcged_u64: 4263 case NEON::BI__builtin_neon_vcged_s64: 4264 case NEON::BI__builtin_neon_vcled_u64: 4265 case NEON::BI__builtin_neon_vcled_s64: { 4266 llvm::CmpInst::Predicate P; 4267 switch (BuiltinID) { 4268 default: llvm_unreachable("missing builtin ID in switch!"); 4269 case NEON::BI__builtin_neon_vceqd_s64: 4270 case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;break; 4271 case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;break; 4272 case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;break; 4273 case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;break; 4274 case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;break; 4275 case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;break; 4276 case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;break; 4277 case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;break; 4278 case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;break; 4279 } 4280 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4281 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); 4282 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); 4283 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]); 4284 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd"); 4285 } 4286 case NEON::BI__builtin_neon_vtstd_s64: 4287 case NEON::BI__builtin_neon_vtstd_u64: { 4288 llvm::Type *Ty = llvm::Type::getInt64Ty(getLLVMContext()); 4289 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4290 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4291 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4292 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); 4293 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], 4294 llvm::Constant::getNullValue(Ty)); 4295 return Builder.CreateSExt(Ops[0], Ty, "vtstd"); 4296 } 4297 case NEON::BI__builtin_neon_vset_lane_i8: 4298 case NEON::BI__builtin_neon_vset_lane_i16: 4299 case NEON::BI__builtin_neon_vset_lane_i32: 4300 case NEON::BI__builtin_neon_vset_lane_i64: 4301 case NEON::BI__builtin_neon_vset_lane_f32: 4302 case NEON::BI__builtin_neon_vsetq_lane_i8: 4303 case NEON::BI__builtin_neon_vsetq_lane_i16: 4304 case NEON::BI__builtin_neon_vsetq_lane_i32: 4305 case NEON::BI__builtin_neon_vsetq_lane_i64: 4306 case NEON::BI__builtin_neon_vsetq_lane_f32: 4307 Ops.push_back(EmitScalarExpr(E->getArg(2))); 4308 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 4309 case NEON::BI__builtin_neon_vset_lane_f64: 4310 // The vector type needs a cast for the v1f64 variant. 4311 Ops[1] = Builder.CreateBitCast(Ops[1], 4312 llvm::VectorType::get(DoubleTy, 1)); 4313 Ops.push_back(EmitScalarExpr(E->getArg(2))); 4314 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 4315 case NEON::BI__builtin_neon_vsetq_lane_f64: 4316 // The vector type needs a cast for the v2f64 variant. 4317 Ops[1] = Builder.CreateBitCast(Ops[1], 4318 llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2)); 4319 Ops.push_back(EmitScalarExpr(E->getArg(2))); 4320 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 4321 4322 case NEON::BI__builtin_neon_vget_lane_i8: 4323 case NEON::BI__builtin_neon_vdupb_lane_i8: 4324 Ops[0] = Builder.CreateBitCast(Ops[0], 4325 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8)); 4326 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4327 "vget_lane"); 4328 case NEON::BI__builtin_neon_vgetq_lane_i8: 4329 case NEON::BI__builtin_neon_vdupb_laneq_i8: 4330 Ops[0] = Builder.CreateBitCast(Ops[0], 4331 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16)); 4332 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4333 "vgetq_lane"); 4334 case NEON::BI__builtin_neon_vget_lane_i16: 4335 case NEON::BI__builtin_neon_vduph_lane_i16: 4336 Ops[0] = Builder.CreateBitCast(Ops[0], 4337 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4)); 4338 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4339 "vget_lane"); 4340 case NEON::BI__builtin_neon_vgetq_lane_i16: 4341 case NEON::BI__builtin_neon_vduph_laneq_i16: 4342 Ops[0] = Builder.CreateBitCast(Ops[0], 4343 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8)); 4344 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4345 "vgetq_lane"); 4346 case NEON::BI__builtin_neon_vget_lane_i32: 4347 case NEON::BI__builtin_neon_vdups_lane_i32: 4348 Ops[0] = Builder.CreateBitCast( 4349 Ops[0], 4350 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 32), 2)); 4351 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4352 "vget_lane"); 4353 case NEON::BI__builtin_neon_vdups_lane_f32: 4354 Ops[0] = Builder.CreateBitCast(Ops[0], 4355 llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2)); 4356 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4357 "vdups_lane"); 4358 case NEON::BI__builtin_neon_vgetq_lane_i32: 4359 case NEON::BI__builtin_neon_vdups_laneq_i32: 4360 Ops[0] = Builder.CreateBitCast(Ops[0], 4361 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 32), 4)); 4362 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4363 "vgetq_lane"); 4364 case NEON::BI__builtin_neon_vget_lane_i64: 4365 case NEON::BI__builtin_neon_vdupd_lane_i64: 4366 Ops[0] = Builder.CreateBitCast(Ops[0], 4367 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 64), 1)); 4368 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4369 "vget_lane"); 4370 case NEON::BI__builtin_neon_vdupd_lane_f64: 4371 Ops[0] = Builder.CreateBitCast(Ops[0], 4372 llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 1)); 4373 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4374 "vdupd_lane"); 4375 case NEON::BI__builtin_neon_vgetq_lane_i64: 4376 case NEON::BI__builtin_neon_vdupd_laneq_i64: 4377 Ops[0] = Builder.CreateBitCast(Ops[0], 4378 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 64), 2)); 4379 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4380 "vgetq_lane"); 4381 case NEON::BI__builtin_neon_vget_lane_f32: 4382 Ops[0] = Builder.CreateBitCast(Ops[0], 4383 llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2)); 4384 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4385 "vget_lane"); 4386 case NEON::BI__builtin_neon_vget_lane_f64: 4387 Ops[0] = Builder.CreateBitCast(Ops[0], 4388 llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 1)); 4389 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4390 "vget_lane"); 4391 case NEON::BI__builtin_neon_vgetq_lane_f32: 4392 case NEON::BI__builtin_neon_vdups_laneq_f32: 4393 Ops[0] = Builder.CreateBitCast(Ops[0], 4394 llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 4)); 4395 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4396 "vgetq_lane"); 4397 case NEON::BI__builtin_neon_vgetq_lane_f64: 4398 case NEON::BI__builtin_neon_vdupd_laneq_f64: 4399 Ops[0] = Builder.CreateBitCast(Ops[0], 4400 llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2)); 4401 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4402 "vgetq_lane"); 4403 case NEON::BI__builtin_neon_vaddd_s64: 4404 case NEON::BI__builtin_neon_vaddd_u64: 4405 return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd"); 4406 case NEON::BI__builtin_neon_vsubd_s64: 4407 case NEON::BI__builtin_neon_vsubd_u64: 4408 return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd"); 4409 case NEON::BI__builtin_neon_vqdmlalh_s16: 4410 case NEON::BI__builtin_neon_vqdmlslh_s16: { 4411 SmallVector<Value *, 2> ProductOps; 4412 ProductOps.push_back(vectorWrapScalar16(Ops[1])); 4413 ProductOps.push_back(vectorWrapScalar16(EmitScalarExpr(E->getArg(2)))); 4414 llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4); 4415 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), 4416 ProductOps, "vqdmlXl"); 4417 Constant *CI = ConstantInt::get(SizeTy, 0); 4418 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); 4419 4420 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16 4421 ? Intrinsic::aarch64_neon_sqadd 4422 : Intrinsic::aarch64_neon_sqsub; 4423 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl"); 4424 } 4425 case NEON::BI__builtin_neon_vqshlud_n_s64: { 4426 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4427 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); 4428 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqshlu, Int64Ty), 4429 Ops, "vqshlu_n"); 4430 } 4431 case NEON::BI__builtin_neon_vqshld_n_u64: 4432 case NEON::BI__builtin_neon_vqshld_n_s64: { 4433 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64 4434 ? Intrinsic::aarch64_neon_uqshl 4435 : Intrinsic::aarch64_neon_sqshl; 4436 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4437 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); 4438 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n"); 4439 } 4440 case NEON::BI__builtin_neon_vrshrd_n_u64: 4441 case NEON::BI__builtin_neon_vrshrd_n_s64: { 4442 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64 4443 ? Intrinsic::aarch64_neon_urshl 4444 : Intrinsic::aarch64_neon_srshl; 4445 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4446 int SV = cast<ConstantInt>(Ops[1])->getSExtValue(); 4447 Ops[1] = ConstantInt::get(Int64Ty, -SV); 4448 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n"); 4449 } 4450 case NEON::BI__builtin_neon_vrsrad_n_u64: 4451 case NEON::BI__builtin_neon_vrsrad_n_s64: { 4452 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64 4453 ? Intrinsic::aarch64_neon_urshl 4454 : Intrinsic::aarch64_neon_srshl; 4455 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); 4456 Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2)))); 4457 Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Int64Ty), Ops[1], 4458 Builder.CreateSExt(Ops[2], Int64Ty)); 4459 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty)); 4460 } 4461 case NEON::BI__builtin_neon_vshld_n_s64: 4462 case NEON::BI__builtin_neon_vshld_n_u64: { 4463 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 4464 return Builder.CreateShl( 4465 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n"); 4466 } 4467 case NEON::BI__builtin_neon_vshrd_n_s64: { 4468 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 4469 return Builder.CreateAShr( 4470 Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), 4471 Amt->getZExtValue())), 4472 "shrd_n"); 4473 } 4474 case NEON::BI__builtin_neon_vshrd_n_u64: { 4475 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 4476 uint64_t ShiftAmt = Amt->getZExtValue(); 4477 // Right-shifting an unsigned value by its size yields 0. 4478 if (ShiftAmt == 64) 4479 return ConstantInt::get(Int64Ty, 0); 4480 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt), 4481 "shrd_n"); 4482 } 4483 case NEON::BI__builtin_neon_vsrad_n_s64: { 4484 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2))); 4485 Ops[1] = Builder.CreateAShr( 4486 Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), 4487 Amt->getZExtValue())), 4488 "shrd_n"); 4489 return Builder.CreateAdd(Ops[0], Ops[1]); 4490 } 4491 case NEON::BI__builtin_neon_vsrad_n_u64: { 4492 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2))); 4493 uint64_t ShiftAmt = Amt->getZExtValue(); 4494 // Right-shifting an unsigned value by its size yields 0. 4495 // As Op + 0 = Op, return Ops[0] directly. 4496 if (ShiftAmt == 64) 4497 return Ops[0]; 4498 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt), 4499 "shrd_n"); 4500 return Builder.CreateAdd(Ops[0], Ops[1]); 4501 } 4502 case NEON::BI__builtin_neon_vqdmlalh_lane_s16: 4503 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16: 4504 case NEON::BI__builtin_neon_vqdmlslh_lane_s16: 4505 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: { 4506 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), 4507 "lane"); 4508 SmallVector<Value *, 2> ProductOps; 4509 ProductOps.push_back(vectorWrapScalar16(Ops[1])); 4510 ProductOps.push_back(vectorWrapScalar16(Ops[2])); 4511 llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4); 4512 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), 4513 ProductOps, "vqdmlXl"); 4514 Constant *CI = ConstantInt::get(SizeTy, 0); 4515 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); 4516 Ops.pop_back(); 4517 4518 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 || 4519 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16) 4520 ? Intrinsic::aarch64_neon_sqadd 4521 : Intrinsic::aarch64_neon_sqsub; 4522 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl"); 4523 } 4524 case NEON::BI__builtin_neon_vqdmlals_s32: 4525 case NEON::BI__builtin_neon_vqdmlsls_s32: { 4526 SmallVector<Value *, 2> ProductOps; 4527 ProductOps.push_back(Ops[1]); 4528 ProductOps.push_back(EmitScalarExpr(E->getArg(2))); 4529 Ops[1] = 4530 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar), 4531 ProductOps, "vqdmlXl"); 4532 4533 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32 4534 ? Intrinsic::aarch64_neon_sqadd 4535 : Intrinsic::aarch64_neon_sqsub; 4536 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl"); 4537 } 4538 case NEON::BI__builtin_neon_vqdmlals_lane_s32: 4539 case NEON::BI__builtin_neon_vqdmlals_laneq_s32: 4540 case NEON::BI__builtin_neon_vqdmlsls_lane_s32: 4541 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: { 4542 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), 4543 "lane"); 4544 SmallVector<Value *, 2> ProductOps; 4545 ProductOps.push_back(Ops[1]); 4546 ProductOps.push_back(Ops[2]); 4547 Ops[1] = 4548 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar), 4549 ProductOps, "vqdmlXl"); 4550 Ops.pop_back(); 4551 4552 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 || 4553 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32) 4554 ? Intrinsic::aarch64_neon_sqadd 4555 : Intrinsic::aarch64_neon_sqsub; 4556 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl"); 4557 } 4558 } 4559 4560 llvm::VectorType *VTy = GetNeonType(this, Type); 4561 llvm::Type *Ty = VTy; 4562 if (!Ty) 4563 return nullptr; 4564 4565 // Not all intrinsics handled by the common case work for AArch64 yet, so only 4566 // defer to common code if it's been added to our special map. 4567 Builtin = findNeonIntrinsicInMap(AArch64SIMDIntrinsicMap, BuiltinID, 4568 AArch64SIMDIntrinsicsProvenSorted); 4569 4570 if (Builtin) 4571 return EmitCommonNeonBuiltinExpr( 4572 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic, 4573 Builtin->NameHint, Builtin->TypeModifier, E, Ops, nullptr); 4574 4575 if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops)) 4576 return V; 4577 4578 unsigned Int; 4579 switch (BuiltinID) { 4580 default: return nullptr; 4581 case NEON::BI__builtin_neon_vbsl_v: 4582 case NEON::BI__builtin_neon_vbslq_v: { 4583 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy); 4584 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl"); 4585 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl"); 4586 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl"); 4587 4588 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl"); 4589 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl"); 4590 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl"); 4591 return Builder.CreateBitCast(Ops[0], Ty); 4592 } 4593 case NEON::BI__builtin_neon_vfma_lane_v: 4594 case NEON::BI__builtin_neon_vfmaq_lane_v: { // Only used for FP types 4595 // The ARM builtins (and instructions) have the addend as the first 4596 // operand, but the 'fma' intrinsics have it last. Swap it around here. 4597 Value *Addend = Ops[0]; 4598 Value *Multiplicand = Ops[1]; 4599 Value *LaneSource = Ops[2]; 4600 Ops[0] = Multiplicand; 4601 Ops[1] = LaneSource; 4602 Ops[2] = Addend; 4603 4604 // Now adjust things to handle the lane access. 4605 llvm::Type *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v ? 4606 llvm::VectorType::get(VTy->getElementType(), VTy->getNumElements() / 2) : 4607 VTy; 4608 llvm::Constant *cst = cast<Constant>(Ops[3]); 4609 Value *SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), cst); 4610 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy); 4611 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane"); 4612 4613 Ops.pop_back(); 4614 Int = Intrinsic::fma; 4615 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla"); 4616 } 4617 case NEON::BI__builtin_neon_vfma_laneq_v: { 4618 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); 4619 // v1f64 fma should be mapped to Neon scalar f64 fma 4620 if (VTy && VTy->getElementType() == DoubleTy) { 4621 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 4622 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); 4623 llvm::Type *VTy = GetNeonType(this, 4624 NeonTypeFlags(NeonTypeFlags::Float64, false, true)); 4625 Ops[2] = Builder.CreateBitCast(Ops[2], VTy); 4626 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); 4627 Value *F = CGM.getIntrinsic(Intrinsic::fma, DoubleTy); 4628 Value *Result = Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]); 4629 return Builder.CreateBitCast(Result, Ty); 4630 } 4631 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 4632 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4633 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4634 4635 llvm::Type *STy = llvm::VectorType::get(VTy->getElementType(), 4636 VTy->getNumElements() * 2); 4637 Ops[2] = Builder.CreateBitCast(Ops[2], STy); 4638 Value* SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), 4639 cast<ConstantInt>(Ops[3])); 4640 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane"); 4641 4642 return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]); 4643 } 4644 case NEON::BI__builtin_neon_vfmaq_laneq_v: { 4645 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 4646 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4647 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4648 4649 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 4650 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3])); 4651 return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]); 4652 } 4653 case NEON::BI__builtin_neon_vfmas_lane_f32: 4654 case NEON::BI__builtin_neon_vfmas_laneq_f32: 4655 case NEON::BI__builtin_neon_vfmad_lane_f64: 4656 case NEON::BI__builtin_neon_vfmad_laneq_f64: { 4657 Ops.push_back(EmitScalarExpr(E->getArg(3))); 4658 llvm::Type *Ty = ConvertType(E->getCallReturnType()); 4659 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 4660 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); 4661 return Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]); 4662 } 4663 case NEON::BI__builtin_neon_vfms_v: 4664 case NEON::BI__builtin_neon_vfmsq_v: { // Only used for FP types 4665 // FIXME: probably remove when we no longer support aarch64_simd.h 4666 // (arm_neon.h delegates to vfma). 4667 4668 // The ARM builtins (and instructions) have the addend as the first 4669 // operand, but the 'fma' intrinsics have it last. Swap it around here. 4670 Value *Subtrahend = Ops[0]; 4671 Value *Multiplicand = Ops[2]; 4672 Ops[0] = Multiplicand; 4673 Ops[2] = Subtrahend; 4674 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); 4675 Ops[1] = Builder.CreateFNeg(Ops[1]); 4676 Int = Intrinsic::fma; 4677 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmls"); 4678 } 4679 case NEON::BI__builtin_neon_vmull_v: 4680 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 4681 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull; 4682 if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull; 4683 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); 4684 case NEON::BI__builtin_neon_vmax_v: 4685 case NEON::BI__builtin_neon_vmaxq_v: 4686 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 4687 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax; 4688 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax; 4689 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax"); 4690 case NEON::BI__builtin_neon_vmin_v: 4691 case NEON::BI__builtin_neon_vminq_v: 4692 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 4693 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin; 4694 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin; 4695 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin"); 4696 case NEON::BI__builtin_neon_vabd_v: 4697 case NEON::BI__builtin_neon_vabdq_v: 4698 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 4699 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd; 4700 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd; 4701 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd"); 4702 case NEON::BI__builtin_neon_vpadal_v: 4703 case NEON::BI__builtin_neon_vpadalq_v: { 4704 unsigned ArgElts = VTy->getNumElements(); 4705 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType()); 4706 unsigned BitWidth = EltTy->getBitWidth(); 4707 llvm::Type *ArgTy = llvm::VectorType::get( 4708 llvm::IntegerType::get(getLLVMContext(), BitWidth/2), 2*ArgElts); 4709 llvm::Type* Tys[2] = { VTy, ArgTy }; 4710 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp; 4711 SmallVector<llvm::Value*, 1> TmpOps; 4712 TmpOps.push_back(Ops[1]); 4713 Function *F = CGM.getIntrinsic(Int, Tys); 4714 llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vpadal"); 4715 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType()); 4716 return Builder.CreateAdd(tmp, addend); 4717 } 4718 case NEON::BI__builtin_neon_vpmin_v: 4719 case NEON::BI__builtin_neon_vpminq_v: 4720 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 4721 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp; 4722 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp; 4723 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin"); 4724 case NEON::BI__builtin_neon_vpmax_v: 4725 case NEON::BI__builtin_neon_vpmaxq_v: 4726 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 4727 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp; 4728 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp; 4729 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax"); 4730 case NEON::BI__builtin_neon_vminnm_v: 4731 case NEON::BI__builtin_neon_vminnmq_v: 4732 Int = Intrinsic::aarch64_neon_fminnm; 4733 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm"); 4734 case NEON::BI__builtin_neon_vmaxnm_v: 4735 case NEON::BI__builtin_neon_vmaxnmq_v: 4736 Int = Intrinsic::aarch64_neon_fmaxnm; 4737 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm"); 4738 case NEON::BI__builtin_neon_vrecpss_f32: { 4739 llvm::Type *f32Type = llvm::Type::getFloatTy(getLLVMContext()); 4740 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4741 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, f32Type), 4742 Ops, "vrecps"); 4743 } 4744 case NEON::BI__builtin_neon_vrecpsd_f64: { 4745 llvm::Type *f64Type = llvm::Type::getDoubleTy(getLLVMContext()); 4746 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4747 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, f64Type), 4748 Ops, "vrecps"); 4749 } 4750 case NEON::BI__builtin_neon_vqshrun_n_v: 4751 Int = Intrinsic::aarch64_neon_sqshrun; 4752 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n"); 4753 case NEON::BI__builtin_neon_vqrshrun_n_v: 4754 Int = Intrinsic::aarch64_neon_sqrshrun; 4755 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n"); 4756 case NEON::BI__builtin_neon_vqshrn_n_v: 4757 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn; 4758 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n"); 4759 case NEON::BI__builtin_neon_vrshrn_n_v: 4760 Int = Intrinsic::aarch64_neon_rshrn; 4761 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n"); 4762 case NEON::BI__builtin_neon_vqrshrn_n_v: 4763 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn; 4764 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n"); 4765 case NEON::BI__builtin_neon_vrnda_v: 4766 case NEON::BI__builtin_neon_vrndaq_v: { 4767 Int = Intrinsic::round; 4768 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda"); 4769 } 4770 case NEON::BI__builtin_neon_vrndi_v: 4771 case NEON::BI__builtin_neon_vrndiq_v: { 4772 Int = Intrinsic::nearbyint; 4773 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndi"); 4774 } 4775 case NEON::BI__builtin_neon_vrndm_v: 4776 case NEON::BI__builtin_neon_vrndmq_v: { 4777 Int = Intrinsic::floor; 4778 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm"); 4779 } 4780 case NEON::BI__builtin_neon_vrndn_v: 4781 case NEON::BI__builtin_neon_vrndnq_v: { 4782 Int = Intrinsic::aarch64_neon_frintn; 4783 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn"); 4784 } 4785 case NEON::BI__builtin_neon_vrndp_v: 4786 case NEON::BI__builtin_neon_vrndpq_v: { 4787 Int = Intrinsic::ceil; 4788 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp"); 4789 } 4790 case NEON::BI__builtin_neon_vrndx_v: 4791 case NEON::BI__builtin_neon_vrndxq_v: { 4792 Int = Intrinsic::rint; 4793 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx"); 4794 } 4795 case NEON::BI__builtin_neon_vrnd_v: 4796 case NEON::BI__builtin_neon_vrndq_v: { 4797 Int = Intrinsic::trunc; 4798 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz"); 4799 } 4800 case NEON::BI__builtin_neon_vceqz_v: 4801 case NEON::BI__builtin_neon_vceqzq_v: 4802 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ, 4803 ICmpInst::ICMP_EQ, "vceqz"); 4804 case NEON::BI__builtin_neon_vcgez_v: 4805 case NEON::BI__builtin_neon_vcgezq_v: 4806 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE, 4807 ICmpInst::ICMP_SGE, "vcgez"); 4808 case NEON::BI__builtin_neon_vclez_v: 4809 case NEON::BI__builtin_neon_vclezq_v: 4810 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE, 4811 ICmpInst::ICMP_SLE, "vclez"); 4812 case NEON::BI__builtin_neon_vcgtz_v: 4813 case NEON::BI__builtin_neon_vcgtzq_v: 4814 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT, 4815 ICmpInst::ICMP_SGT, "vcgtz"); 4816 case NEON::BI__builtin_neon_vcltz_v: 4817 case NEON::BI__builtin_neon_vcltzq_v: 4818 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT, 4819 ICmpInst::ICMP_SLT, "vcltz"); 4820 case NEON::BI__builtin_neon_vcvt_f64_v: 4821 case NEON::BI__builtin_neon_vcvtq_f64_v: 4822 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4823 Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad)); 4824 return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") 4825 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); 4826 case NEON::BI__builtin_neon_vcvt_f64_f32: { 4827 assert(Type.getEltType() == NeonTypeFlags::Float64 && quad && 4828 "unexpected vcvt_f64_f32 builtin"); 4829 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float32, false, false); 4830 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); 4831 4832 return Builder.CreateFPExt(Ops[0], Ty, "vcvt"); 4833 } 4834 case NEON::BI__builtin_neon_vcvt_f32_f64: { 4835 assert(Type.getEltType() == NeonTypeFlags::Float32 && 4836 "unexpected vcvt_f32_f64 builtin"); 4837 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float64, false, true); 4838 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); 4839 4840 return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt"); 4841 } 4842 case NEON::BI__builtin_neon_vcvt_s32_v: 4843 case NEON::BI__builtin_neon_vcvt_u32_v: 4844 case NEON::BI__builtin_neon_vcvt_s64_v: 4845 case NEON::BI__builtin_neon_vcvt_u64_v: 4846 case NEON::BI__builtin_neon_vcvtq_s32_v: 4847 case NEON::BI__builtin_neon_vcvtq_u32_v: 4848 case NEON::BI__builtin_neon_vcvtq_s64_v: 4849 case NEON::BI__builtin_neon_vcvtq_u64_v: { 4850 bool Double = 4851 (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 4852 llvm::Type *InTy = 4853 GetNeonType(this, 4854 NeonTypeFlags(Double ? NeonTypeFlags::Float64 4855 : NeonTypeFlags::Float32, false, quad)); 4856 Ops[0] = Builder.CreateBitCast(Ops[0], InTy); 4857 if (usgn) 4858 return Builder.CreateFPToUI(Ops[0], Ty); 4859 return Builder.CreateFPToSI(Ops[0], Ty); 4860 } 4861 case NEON::BI__builtin_neon_vcvta_s32_v: 4862 case NEON::BI__builtin_neon_vcvtaq_s32_v: 4863 case NEON::BI__builtin_neon_vcvta_u32_v: 4864 case NEON::BI__builtin_neon_vcvtaq_u32_v: 4865 case NEON::BI__builtin_neon_vcvta_s64_v: 4866 case NEON::BI__builtin_neon_vcvtaq_s64_v: 4867 case NEON::BI__builtin_neon_vcvta_u64_v: 4868 case NEON::BI__builtin_neon_vcvtaq_u64_v: { 4869 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas; 4870 bool Double = 4871 (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 4872 llvm::Type *InTy = 4873 GetNeonType(this, 4874 NeonTypeFlags(Double ? NeonTypeFlags::Float64 4875 : NeonTypeFlags::Float32, false, quad)); 4876 llvm::Type *Tys[2] = { Ty, InTy }; 4877 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta"); 4878 } 4879 case NEON::BI__builtin_neon_vcvtm_s32_v: 4880 case NEON::BI__builtin_neon_vcvtmq_s32_v: 4881 case NEON::BI__builtin_neon_vcvtm_u32_v: 4882 case NEON::BI__builtin_neon_vcvtmq_u32_v: 4883 case NEON::BI__builtin_neon_vcvtm_s64_v: 4884 case NEON::BI__builtin_neon_vcvtmq_s64_v: 4885 case NEON::BI__builtin_neon_vcvtm_u64_v: 4886 case NEON::BI__builtin_neon_vcvtmq_u64_v: { 4887 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms; 4888 bool Double = 4889 (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 4890 llvm::Type *InTy = 4891 GetNeonType(this, 4892 NeonTypeFlags(Double ? NeonTypeFlags::Float64 4893 : NeonTypeFlags::Float32, false, quad)); 4894 llvm::Type *Tys[2] = { Ty, InTy }; 4895 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm"); 4896 } 4897 case NEON::BI__builtin_neon_vcvtn_s32_v: 4898 case NEON::BI__builtin_neon_vcvtnq_s32_v: 4899 case NEON::BI__builtin_neon_vcvtn_u32_v: 4900 case NEON::BI__builtin_neon_vcvtnq_u32_v: 4901 case NEON::BI__builtin_neon_vcvtn_s64_v: 4902 case NEON::BI__builtin_neon_vcvtnq_s64_v: 4903 case NEON::BI__builtin_neon_vcvtn_u64_v: 4904 case NEON::BI__builtin_neon_vcvtnq_u64_v: { 4905 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns; 4906 bool Double = 4907 (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 4908 llvm::Type *InTy = 4909 GetNeonType(this, 4910 NeonTypeFlags(Double ? NeonTypeFlags::Float64 4911 : NeonTypeFlags::Float32, false, quad)); 4912 llvm::Type *Tys[2] = { Ty, InTy }; 4913 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn"); 4914 } 4915 case NEON::BI__builtin_neon_vcvtp_s32_v: 4916 case NEON::BI__builtin_neon_vcvtpq_s32_v: 4917 case NEON::BI__builtin_neon_vcvtp_u32_v: 4918 case NEON::BI__builtin_neon_vcvtpq_u32_v: 4919 case NEON::BI__builtin_neon_vcvtp_s64_v: 4920 case NEON::BI__builtin_neon_vcvtpq_s64_v: 4921 case NEON::BI__builtin_neon_vcvtp_u64_v: 4922 case NEON::BI__builtin_neon_vcvtpq_u64_v: { 4923 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps; 4924 bool Double = 4925 (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 4926 llvm::Type *InTy = 4927 GetNeonType(this, 4928 NeonTypeFlags(Double ? NeonTypeFlags::Float64 4929 : NeonTypeFlags::Float32, false, quad)); 4930 llvm::Type *Tys[2] = { Ty, InTy }; 4931 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp"); 4932 } 4933 case NEON::BI__builtin_neon_vmulx_v: 4934 case NEON::BI__builtin_neon_vmulxq_v: { 4935 Int = Intrinsic::aarch64_neon_fmulx; 4936 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx"); 4937 } 4938 case NEON::BI__builtin_neon_vmul_lane_v: 4939 case NEON::BI__builtin_neon_vmul_laneq_v: { 4940 // v1f64 vmul_lane should be mapped to Neon scalar mul lane 4941 bool Quad = false; 4942 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v) 4943 Quad = true; 4944 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 4945 llvm::Type *VTy = GetNeonType(this, 4946 NeonTypeFlags(NeonTypeFlags::Float64, false, Quad)); 4947 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); 4948 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract"); 4949 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]); 4950 return Builder.CreateBitCast(Result, Ty); 4951 } 4952 case NEON::BI__builtin_neon_vnegd_s64: 4953 return Builder.CreateNeg(EmitScalarExpr(E->getArg(0)), "vnegd"); 4954 case NEON::BI__builtin_neon_vpmaxnm_v: 4955 case NEON::BI__builtin_neon_vpmaxnmq_v: { 4956 Int = Intrinsic::aarch64_neon_fmaxnmp; 4957 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm"); 4958 } 4959 case NEON::BI__builtin_neon_vpminnm_v: 4960 case NEON::BI__builtin_neon_vpminnmq_v: { 4961 Int = Intrinsic::aarch64_neon_fminnmp; 4962 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm"); 4963 } 4964 case NEON::BI__builtin_neon_vsqrt_v: 4965 case NEON::BI__builtin_neon_vsqrtq_v: { 4966 Int = Intrinsic::sqrt; 4967 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4968 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt"); 4969 } 4970 case NEON::BI__builtin_neon_vrbit_v: 4971 case NEON::BI__builtin_neon_vrbitq_v: { 4972 Int = Intrinsic::aarch64_neon_rbit; 4973 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit"); 4974 } 4975 case NEON::BI__builtin_neon_vaddv_u8: 4976 // FIXME: These are handled by the AArch64 scalar code. 4977 usgn = true; 4978 // FALLTHROUGH 4979 case NEON::BI__builtin_neon_vaddv_s8: { 4980 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 4981 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 4982 VTy = 4983 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 4984 llvm::Type *Tys[2] = { Ty, VTy }; 4985 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4986 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 4987 return Builder.CreateTrunc(Ops[0], 4988 llvm::IntegerType::get(getLLVMContext(), 8)); 4989 } 4990 case NEON::BI__builtin_neon_vaddv_u16: 4991 usgn = true; 4992 // FALLTHROUGH 4993 case NEON::BI__builtin_neon_vaddv_s16: { 4994 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 4995 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 4996 VTy = 4997 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 4998 llvm::Type *Tys[2] = { Ty, VTy }; 4999 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5000 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 5001 return Builder.CreateTrunc(Ops[0], 5002 llvm::IntegerType::get(getLLVMContext(), 16)); 5003 } 5004 case NEON::BI__builtin_neon_vaddvq_u8: 5005 usgn = true; 5006 // FALLTHROUGH 5007 case NEON::BI__builtin_neon_vaddvq_s8: { 5008 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 5009 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5010 VTy = 5011 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 5012 llvm::Type *Tys[2] = { Ty, VTy }; 5013 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5014 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 5015 return Builder.CreateTrunc(Ops[0], 5016 llvm::IntegerType::get(getLLVMContext(), 8)); 5017 } 5018 case NEON::BI__builtin_neon_vaddvq_u16: 5019 usgn = true; 5020 // FALLTHROUGH 5021 case NEON::BI__builtin_neon_vaddvq_s16: { 5022 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 5023 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5024 VTy = 5025 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 5026 llvm::Type *Tys[2] = { Ty, VTy }; 5027 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5028 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 5029 return Builder.CreateTrunc(Ops[0], 5030 llvm::IntegerType::get(getLLVMContext(), 16)); 5031 } 5032 case NEON::BI__builtin_neon_vmaxv_u8: { 5033 Int = Intrinsic::aarch64_neon_umaxv; 5034 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5035 VTy = 5036 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 5037 llvm::Type *Tys[2] = { Ty, VTy }; 5038 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5039 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5040 return Builder.CreateTrunc(Ops[0], 5041 llvm::IntegerType::get(getLLVMContext(), 8)); 5042 } 5043 case NEON::BI__builtin_neon_vmaxv_u16: { 5044 Int = Intrinsic::aarch64_neon_umaxv; 5045 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5046 VTy = 5047 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 5048 llvm::Type *Tys[2] = { Ty, VTy }; 5049 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5050 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5051 return Builder.CreateTrunc(Ops[0], 5052 llvm::IntegerType::get(getLLVMContext(), 16)); 5053 } 5054 case NEON::BI__builtin_neon_vmaxvq_u8: { 5055 Int = Intrinsic::aarch64_neon_umaxv; 5056 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5057 VTy = 5058 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 5059 llvm::Type *Tys[2] = { Ty, VTy }; 5060 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5061 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5062 return Builder.CreateTrunc(Ops[0], 5063 llvm::IntegerType::get(getLLVMContext(), 8)); 5064 } 5065 case NEON::BI__builtin_neon_vmaxvq_u16: { 5066 Int = Intrinsic::aarch64_neon_umaxv; 5067 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5068 VTy = 5069 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 5070 llvm::Type *Tys[2] = { Ty, VTy }; 5071 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5072 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5073 return Builder.CreateTrunc(Ops[0], 5074 llvm::IntegerType::get(getLLVMContext(), 16)); 5075 } 5076 case NEON::BI__builtin_neon_vmaxv_s8: { 5077 Int = Intrinsic::aarch64_neon_smaxv; 5078 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5079 VTy = 5080 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 5081 llvm::Type *Tys[2] = { Ty, VTy }; 5082 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5083 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5084 return Builder.CreateTrunc(Ops[0], 5085 llvm::IntegerType::get(getLLVMContext(), 8)); 5086 } 5087 case NEON::BI__builtin_neon_vmaxv_s16: { 5088 Int = Intrinsic::aarch64_neon_smaxv; 5089 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5090 VTy = 5091 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 5092 llvm::Type *Tys[2] = { Ty, VTy }; 5093 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5094 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5095 return Builder.CreateTrunc(Ops[0], 5096 llvm::IntegerType::get(getLLVMContext(), 16)); 5097 } 5098 case NEON::BI__builtin_neon_vmaxvq_s8: { 5099 Int = Intrinsic::aarch64_neon_smaxv; 5100 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5101 VTy = 5102 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 5103 llvm::Type *Tys[2] = { Ty, VTy }; 5104 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5105 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5106 return Builder.CreateTrunc(Ops[0], 5107 llvm::IntegerType::get(getLLVMContext(), 8)); 5108 } 5109 case NEON::BI__builtin_neon_vmaxvq_s16: { 5110 Int = Intrinsic::aarch64_neon_smaxv; 5111 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5112 VTy = 5113 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 5114 llvm::Type *Tys[2] = { Ty, VTy }; 5115 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5116 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5117 return Builder.CreateTrunc(Ops[0], 5118 llvm::IntegerType::get(getLLVMContext(), 16)); 5119 } 5120 case NEON::BI__builtin_neon_vminv_u8: { 5121 Int = Intrinsic::aarch64_neon_uminv; 5122 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5123 VTy = 5124 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 5125 llvm::Type *Tys[2] = { Ty, VTy }; 5126 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5127 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5128 return Builder.CreateTrunc(Ops[0], 5129 llvm::IntegerType::get(getLLVMContext(), 8)); 5130 } 5131 case NEON::BI__builtin_neon_vminv_u16: { 5132 Int = Intrinsic::aarch64_neon_uminv; 5133 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5134 VTy = 5135 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 5136 llvm::Type *Tys[2] = { Ty, VTy }; 5137 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5138 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5139 return Builder.CreateTrunc(Ops[0], 5140 llvm::IntegerType::get(getLLVMContext(), 16)); 5141 } 5142 case NEON::BI__builtin_neon_vminvq_u8: { 5143 Int = Intrinsic::aarch64_neon_uminv; 5144 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5145 VTy = 5146 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 5147 llvm::Type *Tys[2] = { Ty, VTy }; 5148 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5149 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5150 return Builder.CreateTrunc(Ops[0], 5151 llvm::IntegerType::get(getLLVMContext(), 8)); 5152 } 5153 case NEON::BI__builtin_neon_vminvq_u16: { 5154 Int = Intrinsic::aarch64_neon_uminv; 5155 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5156 VTy = 5157 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 5158 llvm::Type *Tys[2] = { Ty, VTy }; 5159 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5160 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5161 return Builder.CreateTrunc(Ops[0], 5162 llvm::IntegerType::get(getLLVMContext(), 16)); 5163 } 5164 case NEON::BI__builtin_neon_vminv_s8: { 5165 Int = Intrinsic::aarch64_neon_sminv; 5166 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5167 VTy = 5168 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 5169 llvm::Type *Tys[2] = { Ty, VTy }; 5170 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5171 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5172 return Builder.CreateTrunc(Ops[0], 5173 llvm::IntegerType::get(getLLVMContext(), 8)); 5174 } 5175 case NEON::BI__builtin_neon_vminv_s16: { 5176 Int = Intrinsic::aarch64_neon_sminv; 5177 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5178 VTy = 5179 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 5180 llvm::Type *Tys[2] = { Ty, VTy }; 5181 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5182 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5183 return Builder.CreateTrunc(Ops[0], 5184 llvm::IntegerType::get(getLLVMContext(), 16)); 5185 } 5186 case NEON::BI__builtin_neon_vminvq_s8: { 5187 Int = Intrinsic::aarch64_neon_sminv; 5188 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5189 VTy = 5190 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 5191 llvm::Type *Tys[2] = { Ty, VTy }; 5192 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5193 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5194 return Builder.CreateTrunc(Ops[0], 5195 llvm::IntegerType::get(getLLVMContext(), 8)); 5196 } 5197 case NEON::BI__builtin_neon_vminvq_s16: { 5198 Int = Intrinsic::aarch64_neon_sminv; 5199 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5200 VTy = 5201 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 5202 llvm::Type *Tys[2] = { Ty, VTy }; 5203 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5204 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5205 return Builder.CreateTrunc(Ops[0], 5206 llvm::IntegerType::get(getLLVMContext(), 16)); 5207 } 5208 case NEON::BI__builtin_neon_vmul_n_f64: { 5209 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 5210 Value *RHS = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), DoubleTy); 5211 return Builder.CreateFMul(Ops[0], RHS); 5212 } 5213 case NEON::BI__builtin_neon_vaddlv_u8: { 5214 Int = Intrinsic::aarch64_neon_uaddlv; 5215 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5216 VTy = 5217 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 5218 llvm::Type *Tys[2] = { Ty, VTy }; 5219 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5220 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5221 return Builder.CreateTrunc(Ops[0], 5222 llvm::IntegerType::get(getLLVMContext(), 16)); 5223 } 5224 case NEON::BI__builtin_neon_vaddlv_u16: { 5225 Int = Intrinsic::aarch64_neon_uaddlv; 5226 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5227 VTy = 5228 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 5229 llvm::Type *Tys[2] = { Ty, VTy }; 5230 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5231 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5232 } 5233 case NEON::BI__builtin_neon_vaddlvq_u8: { 5234 Int = Intrinsic::aarch64_neon_uaddlv; 5235 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5236 VTy = 5237 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 5238 llvm::Type *Tys[2] = { Ty, VTy }; 5239 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5240 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5241 return Builder.CreateTrunc(Ops[0], 5242 llvm::IntegerType::get(getLLVMContext(), 16)); 5243 } 5244 case NEON::BI__builtin_neon_vaddlvq_u16: { 5245 Int = Intrinsic::aarch64_neon_uaddlv; 5246 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5247 VTy = 5248 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 5249 llvm::Type *Tys[2] = { Ty, VTy }; 5250 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5251 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5252 } 5253 case NEON::BI__builtin_neon_vaddlv_s8: { 5254 Int = Intrinsic::aarch64_neon_saddlv; 5255 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5256 VTy = 5257 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 5258 llvm::Type *Tys[2] = { Ty, VTy }; 5259 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5260 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5261 return Builder.CreateTrunc(Ops[0], 5262 llvm::IntegerType::get(getLLVMContext(), 16)); 5263 } 5264 case NEON::BI__builtin_neon_vaddlv_s16: { 5265 Int = Intrinsic::aarch64_neon_saddlv; 5266 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5267 VTy = 5268 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 5269 llvm::Type *Tys[2] = { Ty, VTy }; 5270 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5271 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5272 } 5273 case NEON::BI__builtin_neon_vaddlvq_s8: { 5274 Int = Intrinsic::aarch64_neon_saddlv; 5275 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5276 VTy = 5277 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 5278 llvm::Type *Tys[2] = { Ty, VTy }; 5279 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5280 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5281 return Builder.CreateTrunc(Ops[0], 5282 llvm::IntegerType::get(getLLVMContext(), 16)); 5283 } 5284 case NEON::BI__builtin_neon_vaddlvq_s16: { 5285 Int = Intrinsic::aarch64_neon_saddlv; 5286 Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5287 VTy = 5288 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 5289 llvm::Type *Tys[2] = { Ty, VTy }; 5290 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5291 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5292 } 5293 case NEON::BI__builtin_neon_vsri_n_v: 5294 case NEON::BI__builtin_neon_vsriq_n_v: { 5295 Int = Intrinsic::aarch64_neon_vsri; 5296 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty); 5297 return EmitNeonCall(Intrin, Ops, "vsri_n"); 5298 } 5299 case NEON::BI__builtin_neon_vsli_n_v: 5300 case NEON::BI__builtin_neon_vsliq_n_v: { 5301 Int = Intrinsic::aarch64_neon_vsli; 5302 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty); 5303 return EmitNeonCall(Intrin, Ops, "vsli_n"); 5304 } 5305 case NEON::BI__builtin_neon_vsra_n_v: 5306 case NEON::BI__builtin_neon_vsraq_n_v: 5307 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5308 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); 5309 return Builder.CreateAdd(Ops[0], Ops[1]); 5310 case NEON::BI__builtin_neon_vrsra_n_v: 5311 case NEON::BI__builtin_neon_vrsraq_n_v: { 5312 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl; 5313 SmallVector<llvm::Value*,2> TmpOps; 5314 TmpOps.push_back(Ops[1]); 5315 TmpOps.push_back(Ops[2]); 5316 Function* F = CGM.getIntrinsic(Int, Ty); 5317 llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vrshr_n", 1, true); 5318 Ops[0] = Builder.CreateBitCast(Ops[0], VTy); 5319 return Builder.CreateAdd(Ops[0], tmp); 5320 } 5321 // FIXME: Sharing loads & stores with 32-bit is complicated by the absence 5322 // of an Align parameter here. 5323 case NEON::BI__builtin_neon_vld1_x2_v: 5324 case NEON::BI__builtin_neon_vld1q_x2_v: 5325 case NEON::BI__builtin_neon_vld1_x3_v: 5326 case NEON::BI__builtin_neon_vld1q_x3_v: 5327 case NEON::BI__builtin_neon_vld1_x4_v: 5328 case NEON::BI__builtin_neon_vld1q_x4_v: { 5329 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType()); 5330 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5331 llvm::Type *Tys[2] = { VTy, PTy }; 5332 unsigned Int; 5333 switch (BuiltinID) { 5334 case NEON::BI__builtin_neon_vld1_x2_v: 5335 case NEON::BI__builtin_neon_vld1q_x2_v: 5336 Int = Intrinsic::aarch64_neon_ld1x2; 5337 break; 5338 case NEON::BI__builtin_neon_vld1_x3_v: 5339 case NEON::BI__builtin_neon_vld1q_x3_v: 5340 Int = Intrinsic::aarch64_neon_ld1x3; 5341 break; 5342 case NEON::BI__builtin_neon_vld1_x4_v: 5343 case NEON::BI__builtin_neon_vld1q_x4_v: 5344 Int = Intrinsic::aarch64_neon_ld1x4; 5345 break; 5346 } 5347 Function *F = CGM.getIntrinsic(Int, Tys); 5348 Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN"); 5349 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 5350 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5351 return Builder.CreateStore(Ops[1], Ops[0]); 5352 } 5353 case NEON::BI__builtin_neon_vst1_x2_v: 5354 case NEON::BI__builtin_neon_vst1q_x2_v: 5355 case NEON::BI__builtin_neon_vst1_x3_v: 5356 case NEON::BI__builtin_neon_vst1q_x3_v: 5357 case NEON::BI__builtin_neon_vst1_x4_v: 5358 case NEON::BI__builtin_neon_vst1q_x4_v: { 5359 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType()); 5360 llvm::Type *Tys[2] = { VTy, PTy }; 5361 unsigned Int; 5362 switch (BuiltinID) { 5363 case NEON::BI__builtin_neon_vst1_x2_v: 5364 case NEON::BI__builtin_neon_vst1q_x2_v: 5365 Int = Intrinsic::aarch64_neon_st1x2; 5366 break; 5367 case NEON::BI__builtin_neon_vst1_x3_v: 5368 case NEON::BI__builtin_neon_vst1q_x3_v: 5369 Int = Intrinsic::aarch64_neon_st1x3; 5370 break; 5371 case NEON::BI__builtin_neon_vst1_x4_v: 5372 case NEON::BI__builtin_neon_vst1q_x4_v: 5373 Int = Intrinsic::aarch64_neon_st1x4; 5374 break; 5375 } 5376 SmallVector<Value *, 4> IntOps(Ops.begin()+1, Ops.end()); 5377 IntOps.push_back(Ops[0]); 5378 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), IntOps, ""); 5379 } 5380 case NEON::BI__builtin_neon_vld1_v: 5381 case NEON::BI__builtin_neon_vld1q_v: 5382 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); 5383 return Builder.CreateLoad(Ops[0]); 5384 case NEON::BI__builtin_neon_vst1_v: 5385 case NEON::BI__builtin_neon_vst1q_v: 5386 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); 5387 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); 5388 return Builder.CreateStore(Ops[1], Ops[0]); 5389 case NEON::BI__builtin_neon_vld1_lane_v: 5390 case NEON::BI__builtin_neon_vld1q_lane_v: 5391 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5392 Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 5393 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5394 Ops[0] = Builder.CreateLoad(Ops[0]); 5395 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane"); 5396 case NEON::BI__builtin_neon_vld1_dup_v: 5397 case NEON::BI__builtin_neon_vld1q_dup_v: { 5398 Value *V = UndefValue::get(Ty); 5399 Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 5400 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5401 Ops[0] = Builder.CreateLoad(Ops[0]); 5402 llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); 5403 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI); 5404 return EmitNeonSplat(Ops[0], CI); 5405 } 5406 case NEON::BI__builtin_neon_vst1_lane_v: 5407 case NEON::BI__builtin_neon_vst1q_lane_v: 5408 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5409 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); 5410 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 5411 return Builder.CreateStore(Ops[1], Builder.CreateBitCast(Ops[0], Ty)); 5412 case NEON::BI__builtin_neon_vld2_v: 5413 case NEON::BI__builtin_neon_vld2q_v: { 5414 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 5415 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5416 llvm::Type *Tys[2] = { VTy, PTy }; 5417 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys); 5418 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); 5419 Ops[0] = Builder.CreateBitCast(Ops[0], 5420 llvm::PointerType::getUnqual(Ops[1]->getType())); 5421 return Builder.CreateStore(Ops[1], Ops[0]); 5422 } 5423 case NEON::BI__builtin_neon_vld3_v: 5424 case NEON::BI__builtin_neon_vld3q_v: { 5425 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 5426 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5427 llvm::Type *Tys[2] = { VTy, PTy }; 5428 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys); 5429 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); 5430 Ops[0] = Builder.CreateBitCast(Ops[0], 5431 llvm::PointerType::getUnqual(Ops[1]->getType())); 5432 return Builder.CreateStore(Ops[1], Ops[0]); 5433 } 5434 case NEON::BI__builtin_neon_vld4_v: 5435 case NEON::BI__builtin_neon_vld4q_v: { 5436 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 5437 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5438 llvm::Type *Tys[2] = { VTy, PTy }; 5439 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys); 5440 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); 5441 Ops[0] = Builder.CreateBitCast(Ops[0], 5442 llvm::PointerType::getUnqual(Ops[1]->getType())); 5443 return Builder.CreateStore(Ops[1], Ops[0]); 5444 } 5445 case NEON::BI__builtin_neon_vld2_dup_v: 5446 case NEON::BI__builtin_neon_vld2q_dup_v: { 5447 llvm::Type *PTy = 5448 llvm::PointerType::getUnqual(VTy->getElementType()); 5449 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5450 llvm::Type *Tys[2] = { VTy, PTy }; 5451 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys); 5452 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); 5453 Ops[0] = Builder.CreateBitCast(Ops[0], 5454 llvm::PointerType::getUnqual(Ops[1]->getType())); 5455 return Builder.CreateStore(Ops[1], Ops[0]); 5456 } 5457 case NEON::BI__builtin_neon_vld3_dup_v: 5458 case NEON::BI__builtin_neon_vld3q_dup_v: { 5459 llvm::Type *PTy = 5460 llvm::PointerType::getUnqual(VTy->getElementType()); 5461 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5462 llvm::Type *Tys[2] = { VTy, PTy }; 5463 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys); 5464 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); 5465 Ops[0] = Builder.CreateBitCast(Ops[0], 5466 llvm::PointerType::getUnqual(Ops[1]->getType())); 5467 return Builder.CreateStore(Ops[1], Ops[0]); 5468 } 5469 case NEON::BI__builtin_neon_vld4_dup_v: 5470 case NEON::BI__builtin_neon_vld4q_dup_v: { 5471 llvm::Type *PTy = 5472 llvm::PointerType::getUnqual(VTy->getElementType()); 5473 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5474 llvm::Type *Tys[2] = { VTy, PTy }; 5475 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys); 5476 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); 5477 Ops[0] = Builder.CreateBitCast(Ops[0], 5478 llvm::PointerType::getUnqual(Ops[1]->getType())); 5479 return Builder.CreateStore(Ops[1], Ops[0]); 5480 } 5481 case NEON::BI__builtin_neon_vld2_lane_v: 5482 case NEON::BI__builtin_neon_vld2q_lane_v: { 5483 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 5484 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys); 5485 Ops.push_back(Ops[1]); 5486 Ops.erase(Ops.begin()+1); 5487 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5488 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 5489 Ops[3] = Builder.CreateZExt(Ops[3], 5490 llvm::IntegerType::get(getLLVMContext(), 64)); 5491 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane"); 5492 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 5493 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5494 return Builder.CreateStore(Ops[1], Ops[0]); 5495 } 5496 case NEON::BI__builtin_neon_vld3_lane_v: 5497 case NEON::BI__builtin_neon_vld3q_lane_v: { 5498 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 5499 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys); 5500 Ops.push_back(Ops[1]); 5501 Ops.erase(Ops.begin()+1); 5502 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5503 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 5504 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); 5505 Ops[4] = Builder.CreateZExt(Ops[4], 5506 llvm::IntegerType::get(getLLVMContext(), 64)); 5507 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane"); 5508 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 5509 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5510 return Builder.CreateStore(Ops[1], Ops[0]); 5511 } 5512 case NEON::BI__builtin_neon_vld4_lane_v: 5513 case NEON::BI__builtin_neon_vld4q_lane_v: { 5514 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 5515 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys); 5516 Ops.push_back(Ops[1]); 5517 Ops.erase(Ops.begin()+1); 5518 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5519 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 5520 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); 5521 Ops[4] = Builder.CreateBitCast(Ops[4], Ty); 5522 Ops[5] = Builder.CreateZExt(Ops[5], 5523 llvm::IntegerType::get(getLLVMContext(), 64)); 5524 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld4_lane"); 5525 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 5526 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5527 return Builder.CreateStore(Ops[1], Ops[0]); 5528 } 5529 case NEON::BI__builtin_neon_vst2_v: 5530 case NEON::BI__builtin_neon_vst2q_v: { 5531 Ops.push_back(Ops[0]); 5532 Ops.erase(Ops.begin()); 5533 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() }; 5534 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys), 5535 Ops, ""); 5536 } 5537 case NEON::BI__builtin_neon_vst2_lane_v: 5538 case NEON::BI__builtin_neon_vst2q_lane_v: { 5539 Ops.push_back(Ops[0]); 5540 Ops.erase(Ops.begin()); 5541 Ops[2] = Builder.CreateZExt(Ops[2], 5542 llvm::IntegerType::get(getLLVMContext(), 64)); 5543 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; 5544 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys), 5545 Ops, ""); 5546 } 5547 case NEON::BI__builtin_neon_vst3_v: 5548 case NEON::BI__builtin_neon_vst3q_v: { 5549 Ops.push_back(Ops[0]); 5550 Ops.erase(Ops.begin()); 5551 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; 5552 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys), 5553 Ops, ""); 5554 } 5555 case NEON::BI__builtin_neon_vst3_lane_v: 5556 case NEON::BI__builtin_neon_vst3q_lane_v: { 5557 Ops.push_back(Ops[0]); 5558 Ops.erase(Ops.begin()); 5559 Ops[3] = Builder.CreateZExt(Ops[3], 5560 llvm::IntegerType::get(getLLVMContext(), 64)); 5561 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; 5562 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys), 5563 Ops, ""); 5564 } 5565 case NEON::BI__builtin_neon_vst4_v: 5566 case NEON::BI__builtin_neon_vst4q_v: { 5567 Ops.push_back(Ops[0]); 5568 Ops.erase(Ops.begin()); 5569 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; 5570 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys), 5571 Ops, ""); 5572 } 5573 case NEON::BI__builtin_neon_vst4_lane_v: 5574 case NEON::BI__builtin_neon_vst4q_lane_v: { 5575 Ops.push_back(Ops[0]); 5576 Ops.erase(Ops.begin()); 5577 Ops[4] = Builder.CreateZExt(Ops[4], 5578 llvm::IntegerType::get(getLLVMContext(), 64)); 5579 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() }; 5580 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys), 5581 Ops, ""); 5582 } 5583 case NEON::BI__builtin_neon_vtrn_v: 5584 case NEON::BI__builtin_neon_vtrnq_v: { 5585 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 5586 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5587 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 5588 Value *SV = nullptr; 5589 5590 for (unsigned vi = 0; vi != 2; ++vi) { 5591 SmallVector<Constant*, 16> Indices; 5592 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 5593 Indices.push_back(ConstantInt::get(Int32Ty, i+vi)); 5594 Indices.push_back(ConstantInt::get(Int32Ty, i+e+vi)); 5595 } 5596 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); 5597 SV = llvm::ConstantVector::get(Indices); 5598 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn"); 5599 SV = Builder.CreateStore(SV, Addr); 5600 } 5601 return SV; 5602 } 5603 case NEON::BI__builtin_neon_vuzp_v: 5604 case NEON::BI__builtin_neon_vuzpq_v: { 5605 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 5606 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5607 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 5608 Value *SV = nullptr; 5609 5610 for (unsigned vi = 0; vi != 2; ++vi) { 5611 SmallVector<Constant*, 16> Indices; 5612 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 5613 Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi)); 5614 5615 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); 5616 SV = llvm::ConstantVector::get(Indices); 5617 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp"); 5618 SV = Builder.CreateStore(SV, Addr); 5619 } 5620 return SV; 5621 } 5622 case NEON::BI__builtin_neon_vzip_v: 5623 case NEON::BI__builtin_neon_vzipq_v: { 5624 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 5625 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5626 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 5627 Value *SV = nullptr; 5628 5629 for (unsigned vi = 0; vi != 2; ++vi) { 5630 SmallVector<Constant*, 16> Indices; 5631 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 5632 Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1)); 5633 Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e)); 5634 } 5635 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); 5636 SV = llvm::ConstantVector::get(Indices); 5637 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip"); 5638 SV = Builder.CreateStore(SV, Addr); 5639 } 5640 return SV; 5641 } 5642 case NEON::BI__builtin_neon_vqtbl1q_v: { 5643 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty), 5644 Ops, "vtbl1"); 5645 } 5646 case NEON::BI__builtin_neon_vqtbl2q_v: { 5647 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty), 5648 Ops, "vtbl2"); 5649 } 5650 case NEON::BI__builtin_neon_vqtbl3q_v: { 5651 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty), 5652 Ops, "vtbl3"); 5653 } 5654 case NEON::BI__builtin_neon_vqtbl4q_v: { 5655 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty), 5656 Ops, "vtbl4"); 5657 } 5658 case NEON::BI__builtin_neon_vqtbx1q_v: { 5659 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty), 5660 Ops, "vtbx1"); 5661 } 5662 case NEON::BI__builtin_neon_vqtbx2q_v: { 5663 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty), 5664 Ops, "vtbx2"); 5665 } 5666 case NEON::BI__builtin_neon_vqtbx3q_v: { 5667 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty), 5668 Ops, "vtbx3"); 5669 } 5670 case NEON::BI__builtin_neon_vqtbx4q_v: { 5671 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty), 5672 Ops, "vtbx4"); 5673 } 5674 case NEON::BI__builtin_neon_vsqadd_v: 5675 case NEON::BI__builtin_neon_vsqaddq_v: { 5676 Int = Intrinsic::aarch64_neon_usqadd; 5677 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd"); 5678 } 5679 case NEON::BI__builtin_neon_vuqadd_v: 5680 case NEON::BI__builtin_neon_vuqaddq_v: { 5681 Int = Intrinsic::aarch64_neon_suqadd; 5682 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd"); 5683 } 5684 } 5685 } 5686 5687 llvm::Value *CodeGenFunction:: 5688 BuildVector(ArrayRef<llvm::Value*> Ops) { 5689 assert((Ops.size() & (Ops.size() - 1)) == 0 && 5690 "Not a power-of-two sized vector!"); 5691 bool AllConstants = true; 5692 for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i) 5693 AllConstants &= isa<Constant>(Ops[i]); 5694 5695 // If this is a constant vector, create a ConstantVector. 5696 if (AllConstants) { 5697 SmallVector<llvm::Constant*, 16> CstOps; 5698 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 5699 CstOps.push_back(cast<Constant>(Ops[i])); 5700 return llvm::ConstantVector::get(CstOps); 5701 } 5702 5703 // Otherwise, insertelement the values to build the vector. 5704 Value *Result = 5705 llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size())); 5706 5707 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 5708 Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i)); 5709 5710 return Result; 5711 } 5712 5713 Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, 5714 const CallExpr *E) { 5715 SmallVector<Value*, 4> Ops; 5716 5717 // Find out if any arguments are required to be integer constant expressions. 5718 unsigned ICEArguments = 0; 5719 ASTContext::GetBuiltinTypeError Error; 5720 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 5721 assert(Error == ASTContext::GE_None && "Should not codegen an error"); 5722 5723 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) { 5724 // If this is a normal argument, just emit it as a scalar. 5725 if ((ICEArguments & (1 << i)) == 0) { 5726 Ops.push_back(EmitScalarExpr(E->getArg(i))); 5727 continue; 5728 } 5729 5730 // If this is required to be a constant, constant fold it so that we know 5731 // that the generated intrinsic gets a ConstantInt. 5732 llvm::APSInt Result; 5733 bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext()); 5734 assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst; 5735 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); 5736 } 5737 5738 switch (BuiltinID) { 5739 default: return nullptr; 5740 case X86::BI_mm_prefetch: { 5741 Value *Address = EmitScalarExpr(E->getArg(0)); 5742 Value *RW = ConstantInt::get(Int32Ty, 0); 5743 Value *Locality = EmitScalarExpr(E->getArg(1)); 5744 Value *Data = ConstantInt::get(Int32Ty, 1); 5745 Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 5746 return Builder.CreateCall4(F, Address, RW, Locality, Data); 5747 } 5748 case X86::BI__builtin_ia32_vec_init_v8qi: 5749 case X86::BI__builtin_ia32_vec_init_v4hi: 5750 case X86::BI__builtin_ia32_vec_init_v2si: 5751 return Builder.CreateBitCast(BuildVector(Ops), 5752 llvm::Type::getX86_MMXTy(getLLVMContext())); 5753 case X86::BI__builtin_ia32_vec_ext_v2si: 5754 return Builder.CreateExtractElement(Ops[0], 5755 llvm::ConstantInt::get(Ops[1]->getType(), 0)); 5756 case X86::BI__builtin_ia32_ldmxcsr: { 5757 Value *Tmp = CreateMemTemp(E->getArg(0)->getType()); 5758 Builder.CreateStore(Ops[0], Tmp); 5759 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr), 5760 Builder.CreateBitCast(Tmp, Int8PtrTy)); 5761 } 5762 case X86::BI__builtin_ia32_stmxcsr: { 5763 Value *Tmp = CreateMemTemp(E->getType()); 5764 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr), 5765 Builder.CreateBitCast(Tmp, Int8PtrTy)); 5766 return Builder.CreateLoad(Tmp, "stmxcsr"); 5767 } 5768 case X86::BI__builtin_ia32_storehps: 5769 case X86::BI__builtin_ia32_storelps: { 5770 llvm::Type *PtrTy = llvm::PointerType::getUnqual(Int64Ty); 5771 llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2); 5772 5773 // cast val v2i64 5774 Ops[1] = Builder.CreateBitCast(Ops[1], VecTy, "cast"); 5775 5776 // extract (0, 1) 5777 unsigned Index = BuiltinID == X86::BI__builtin_ia32_storelps ? 0 : 1; 5778 llvm::Value *Idx = llvm::ConstantInt::get(SizeTy, Index); 5779 Ops[1] = Builder.CreateExtractElement(Ops[1], Idx, "extract"); 5780 5781 // cast pointer to i64 & store 5782 Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy); 5783 return Builder.CreateStore(Ops[1], Ops[0]); 5784 } 5785 case X86::BI__builtin_ia32_palignr: { 5786 unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); 5787 5788 // If palignr is shifting the pair of input vectors less than 9 bytes, 5789 // emit a shuffle instruction. 5790 if (shiftVal <= 8) { 5791 SmallVector<llvm::Constant*, 8> Indices; 5792 for (unsigned i = 0; i != 8; ++i) 5793 Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i)); 5794 5795 Value* SV = llvm::ConstantVector::get(Indices); 5796 return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); 5797 } 5798 5799 // If palignr is shifting the pair of input vectors more than 8 but less 5800 // than 16 bytes, emit a logical right shift of the destination. 5801 if (shiftVal < 16) { 5802 // MMX has these as 1 x i64 vectors for some odd optimization reasons. 5803 llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 1); 5804 5805 Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); 5806 Ops[1] = llvm::ConstantInt::get(VecTy, (shiftVal-8) * 8); 5807 5808 // create i32 constant 5809 llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_mmx_psrl_q); 5810 return Builder.CreateCall(F, makeArrayRef(Ops.data(), 2), "palignr"); 5811 } 5812 5813 // If palignr is shifting the pair of vectors more than 16 bytes, emit zero. 5814 return llvm::Constant::getNullValue(ConvertType(E->getType())); 5815 } 5816 case X86::BI__builtin_ia32_palignr128: { 5817 unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); 5818 5819 // If palignr is shifting the pair of input vectors less than 17 bytes, 5820 // emit a shuffle instruction. 5821 if (shiftVal <= 16) { 5822 SmallVector<llvm::Constant*, 16> Indices; 5823 for (unsigned i = 0; i != 16; ++i) 5824 Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i)); 5825 5826 Value* SV = llvm::ConstantVector::get(Indices); 5827 return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); 5828 } 5829 5830 // If palignr is shifting the pair of input vectors more than 16 but less 5831 // than 32 bytes, emit a logical right shift of the destination. 5832 if (shiftVal < 32) { 5833 llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2); 5834 5835 Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); 5836 Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8); 5837 5838 // create i32 constant 5839 llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_sse2_psrl_dq); 5840 return Builder.CreateCall(F, makeArrayRef(Ops.data(), 2), "palignr"); 5841 } 5842 5843 // If palignr is shifting the pair of vectors more than 32 bytes, emit zero. 5844 return llvm::Constant::getNullValue(ConvertType(E->getType())); 5845 } 5846 case X86::BI__builtin_ia32_palignr256: { 5847 unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); 5848 5849 // If palignr is shifting the pair of input vectors less than 17 bytes, 5850 // emit a shuffle instruction. 5851 if (shiftVal <= 16) { 5852 SmallVector<llvm::Constant*, 32> Indices; 5853 // 256-bit palignr operates on 128-bit lanes so we need to handle that 5854 for (unsigned l = 0; l != 2; ++l) { 5855 unsigned LaneStart = l * 16; 5856 unsigned LaneEnd = (l+1) * 16; 5857 for (unsigned i = 0; i != 16; ++i) { 5858 unsigned Idx = shiftVal + i + LaneStart; 5859 if (Idx >= LaneEnd) Idx += 16; // end of lane, switch operand 5860 Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx)); 5861 } 5862 } 5863 5864 Value* SV = llvm::ConstantVector::get(Indices); 5865 return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); 5866 } 5867 5868 // If palignr is shifting the pair of input vectors more than 16 but less 5869 // than 32 bytes, emit a logical right shift of the destination. 5870 if (shiftVal < 32) { 5871 llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 4); 5872 5873 Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); 5874 Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8); 5875 5876 // create i32 constant 5877 llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_avx2_psrl_dq); 5878 return Builder.CreateCall(F, makeArrayRef(Ops.data(), 2), "palignr"); 5879 } 5880 5881 // If palignr is shifting the pair of vectors more than 32 bytes, emit zero. 5882 return llvm::Constant::getNullValue(ConvertType(E->getType())); 5883 } 5884 case X86::BI__builtin_ia32_movntps: 5885 case X86::BI__builtin_ia32_movntps256: 5886 case X86::BI__builtin_ia32_movntpd: 5887 case X86::BI__builtin_ia32_movntpd256: 5888 case X86::BI__builtin_ia32_movntdq: 5889 case X86::BI__builtin_ia32_movntdq256: 5890 case X86::BI__builtin_ia32_movnti: 5891 case X86::BI__builtin_ia32_movnti64: { 5892 llvm::MDNode *Node = llvm::MDNode::get(getLLVMContext(), 5893 Builder.getInt32(1)); 5894 5895 // Convert the type of the pointer to a pointer to the stored type. 5896 Value *BC = Builder.CreateBitCast(Ops[0], 5897 llvm::PointerType::getUnqual(Ops[1]->getType()), 5898 "cast"); 5899 StoreInst *SI = Builder.CreateStore(Ops[1], BC); 5900 SI->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node); 5901 5902 // If the operand is an integer, we can't assume alignment. Otherwise, 5903 // assume natural alignment. 5904 QualType ArgTy = E->getArg(1)->getType(); 5905 unsigned Align; 5906 if (ArgTy->isIntegerType()) 5907 Align = 1; 5908 else 5909 Align = getContext().getTypeSizeInChars(ArgTy).getQuantity(); 5910 SI->setAlignment(Align); 5911 return SI; 5912 } 5913 // 3DNow! 5914 case X86::BI__builtin_ia32_pswapdsf: 5915 case X86::BI__builtin_ia32_pswapdsi: { 5916 const char *name = nullptr; 5917 Intrinsic::ID ID = Intrinsic::not_intrinsic; 5918 switch(BuiltinID) { 5919 default: llvm_unreachable("Unsupported intrinsic!"); 5920 case X86::BI__builtin_ia32_pswapdsf: 5921 case X86::BI__builtin_ia32_pswapdsi: 5922 name = "pswapd"; 5923 ID = Intrinsic::x86_3dnowa_pswapd; 5924 break; 5925 } 5926 llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext()); 5927 Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast"); 5928 llvm::Function *F = CGM.getIntrinsic(ID); 5929 return Builder.CreateCall(F, Ops, name); 5930 } 5931 case X86::BI__builtin_ia32_rdrand16_step: 5932 case X86::BI__builtin_ia32_rdrand32_step: 5933 case X86::BI__builtin_ia32_rdrand64_step: 5934 case X86::BI__builtin_ia32_rdseed16_step: 5935 case X86::BI__builtin_ia32_rdseed32_step: 5936 case X86::BI__builtin_ia32_rdseed64_step: { 5937 Intrinsic::ID ID; 5938 switch (BuiltinID) { 5939 default: llvm_unreachable("Unsupported intrinsic!"); 5940 case X86::BI__builtin_ia32_rdrand16_step: 5941 ID = Intrinsic::x86_rdrand_16; 5942 break; 5943 case X86::BI__builtin_ia32_rdrand32_step: 5944 ID = Intrinsic::x86_rdrand_32; 5945 break; 5946 case X86::BI__builtin_ia32_rdrand64_step: 5947 ID = Intrinsic::x86_rdrand_64; 5948 break; 5949 case X86::BI__builtin_ia32_rdseed16_step: 5950 ID = Intrinsic::x86_rdseed_16; 5951 break; 5952 case X86::BI__builtin_ia32_rdseed32_step: 5953 ID = Intrinsic::x86_rdseed_32; 5954 break; 5955 case X86::BI__builtin_ia32_rdseed64_step: 5956 ID = Intrinsic::x86_rdseed_64; 5957 break; 5958 } 5959 5960 Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID)); 5961 Builder.CreateStore(Builder.CreateExtractValue(Call, 0), Ops[0]); 5962 return Builder.CreateExtractValue(Call, 1); 5963 } 5964 // AVX2 broadcast 5965 case X86::BI__builtin_ia32_vbroadcastsi256: { 5966 Value *VecTmp = CreateMemTemp(E->getArg(0)->getType()); 5967 Builder.CreateStore(Ops[0], VecTmp); 5968 Value *F = CGM.getIntrinsic(Intrinsic::x86_avx2_vbroadcasti128); 5969 return Builder.CreateCall(F, Builder.CreateBitCast(VecTmp, Int8PtrTy)); 5970 } 5971 } 5972 } 5973 5974 5975 Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, 5976 const CallExpr *E) { 5977 SmallVector<Value*, 4> Ops; 5978 5979 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) 5980 Ops.push_back(EmitScalarExpr(E->getArg(i))); 5981 5982 Intrinsic::ID ID = Intrinsic::not_intrinsic; 5983 5984 switch (BuiltinID) { 5985 default: return nullptr; 5986 5987 // vec_ld, vec_lvsl, vec_lvsr 5988 case PPC::BI__builtin_altivec_lvx: 5989 case PPC::BI__builtin_altivec_lvxl: 5990 case PPC::BI__builtin_altivec_lvebx: 5991 case PPC::BI__builtin_altivec_lvehx: 5992 case PPC::BI__builtin_altivec_lvewx: 5993 case PPC::BI__builtin_altivec_lvsl: 5994 case PPC::BI__builtin_altivec_lvsr: 5995 { 5996 Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); 5997 5998 Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]); 5999 Ops.pop_back(); 6000 6001 switch (BuiltinID) { 6002 default: llvm_unreachable("Unsupported ld/lvsl/lvsr intrinsic!"); 6003 case PPC::BI__builtin_altivec_lvx: 6004 ID = Intrinsic::ppc_altivec_lvx; 6005 break; 6006 case PPC::BI__builtin_altivec_lvxl: 6007 ID = Intrinsic::ppc_altivec_lvxl; 6008 break; 6009 case PPC::BI__builtin_altivec_lvebx: 6010 ID = Intrinsic::ppc_altivec_lvebx; 6011 break; 6012 case PPC::BI__builtin_altivec_lvehx: 6013 ID = Intrinsic::ppc_altivec_lvehx; 6014 break; 6015 case PPC::BI__builtin_altivec_lvewx: 6016 ID = Intrinsic::ppc_altivec_lvewx; 6017 break; 6018 case PPC::BI__builtin_altivec_lvsl: 6019 ID = Intrinsic::ppc_altivec_lvsl; 6020 break; 6021 case PPC::BI__builtin_altivec_lvsr: 6022 ID = Intrinsic::ppc_altivec_lvsr; 6023 break; 6024 } 6025 llvm::Function *F = CGM.getIntrinsic(ID); 6026 return Builder.CreateCall(F, Ops, ""); 6027 } 6028 6029 // vec_st 6030 case PPC::BI__builtin_altivec_stvx: 6031 case PPC::BI__builtin_altivec_stvxl: 6032 case PPC::BI__builtin_altivec_stvebx: 6033 case PPC::BI__builtin_altivec_stvehx: 6034 case PPC::BI__builtin_altivec_stvewx: 6035 { 6036 Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy); 6037 Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]); 6038 Ops.pop_back(); 6039 6040 switch (BuiltinID) { 6041 default: llvm_unreachable("Unsupported st intrinsic!"); 6042 case PPC::BI__builtin_altivec_stvx: 6043 ID = Intrinsic::ppc_altivec_stvx; 6044 break; 6045 case PPC::BI__builtin_altivec_stvxl: 6046 ID = Intrinsic::ppc_altivec_stvxl; 6047 break; 6048 case PPC::BI__builtin_altivec_stvebx: 6049 ID = Intrinsic::ppc_altivec_stvebx; 6050 break; 6051 case PPC::BI__builtin_altivec_stvehx: 6052 ID = Intrinsic::ppc_altivec_stvehx; 6053 break; 6054 case PPC::BI__builtin_altivec_stvewx: 6055 ID = Intrinsic::ppc_altivec_stvewx; 6056 break; 6057 } 6058 llvm::Function *F = CGM.getIntrinsic(ID); 6059 return Builder.CreateCall(F, Ops, ""); 6060 } 6061 } 6062 } 6063 6064 // Emit an intrinsic that has 1 float or double. 6065 static Value *emitUnaryFPBuiltin(CodeGenFunction &CGF, 6066 const CallExpr *E, 6067 unsigned IntrinsicID) { 6068 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); 6069 6070 Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); 6071 return CGF.Builder.CreateCall(F, Src0); 6072 } 6073 6074 // Emit an intrinsic that has 3 float or double operands. 6075 static Value *emitTernaryFPBuiltin(CodeGenFunction &CGF, 6076 const CallExpr *E, 6077 unsigned IntrinsicID) { 6078 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); 6079 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); 6080 llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2)); 6081 6082 Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); 6083 return CGF.Builder.CreateCall3(F, Src0, Src1, Src2); 6084 } 6085 6086 // Emit an intrinsic that has 1 float or double operand, and 1 integer. 6087 static Value *emitFPIntBuiltin(CodeGenFunction &CGF, 6088 const CallExpr *E, 6089 unsigned IntrinsicID) { 6090 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); 6091 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); 6092 6093 Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); 6094 return CGF.Builder.CreateCall2(F, Src0, Src1); 6095 } 6096 6097 Value *CodeGenFunction::EmitR600BuiltinExpr(unsigned BuiltinID, 6098 const CallExpr *E) { 6099 switch (BuiltinID) { 6100 case R600::BI__builtin_amdgpu_div_scale: 6101 case R600::BI__builtin_amdgpu_div_scalef: { 6102 // Translate from the intrinsics's struct return to the builtin's out 6103 // argument. 6104 6105 std::pair<llvm::Value *, unsigned> FlagOutPtr 6106 = EmitPointerWithAlignment(E->getArg(3)); 6107 6108 llvm::Value *X = EmitScalarExpr(E->getArg(0)); 6109 llvm::Value *Y = EmitScalarExpr(E->getArg(1)); 6110 llvm::Value *Z = EmitScalarExpr(E->getArg(2)); 6111 6112 llvm::Value *Callee = CGM.getIntrinsic(Intrinsic::AMDGPU_div_scale, 6113 X->getType()); 6114 6115 llvm::Value *Tmp = Builder.CreateCall3(Callee, X, Y, Z); 6116 6117 llvm::Value *Result = Builder.CreateExtractValue(Tmp, 0); 6118 llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1); 6119 6120 llvm::Type *RealFlagType 6121 = FlagOutPtr.first->getType()->getPointerElementType(); 6122 6123 llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType); 6124 llvm::StoreInst *FlagStore = Builder.CreateStore(FlagExt, FlagOutPtr.first); 6125 FlagStore->setAlignment(FlagOutPtr.second); 6126 return Result; 6127 } 6128 case R600::BI__builtin_amdgpu_div_fmas: 6129 case R600::BI__builtin_amdgpu_div_fmasf: 6130 return emitTernaryFPBuiltin(*this, E, Intrinsic::AMDGPU_div_fmas); 6131 case R600::BI__builtin_amdgpu_div_fixup: 6132 case R600::BI__builtin_amdgpu_div_fixupf: 6133 return emitTernaryFPBuiltin(*this, E, Intrinsic::AMDGPU_div_fixup); 6134 case R600::BI__builtin_amdgpu_trig_preop: 6135 case R600::BI__builtin_amdgpu_trig_preopf: 6136 return emitFPIntBuiltin(*this, E, Intrinsic::AMDGPU_trig_preop); 6137 case R600::BI__builtin_amdgpu_rcp: 6138 case R600::BI__builtin_amdgpu_rcpf: 6139 return emitUnaryFPBuiltin(*this, E, Intrinsic::AMDGPU_rcp); 6140 case R600::BI__builtin_amdgpu_rsq: 6141 case R600::BI__builtin_amdgpu_rsqf: 6142 return emitUnaryFPBuiltin(*this, E, Intrinsic::AMDGPU_rsq); 6143 case R600::BI__builtin_amdgpu_rsq_clamped: 6144 case R600::BI__builtin_amdgpu_rsq_clampedf: 6145 return emitUnaryFPBuiltin(*this, E, Intrinsic::AMDGPU_rsq_clamped); 6146 case R600::BI__builtin_amdgpu_ldexp: 6147 case R600::BI__builtin_amdgpu_ldexpf: 6148 return emitFPIntBuiltin(*this, E, Intrinsic::AMDGPU_ldexp); 6149 default: 6150 return nullptr; 6151 } 6152 } 6153