Lines Matching refs:Ops
4141 Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops, in EmitNeonCall() argument
4148 Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift); in EmitNeonCall()
4150 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name); in EmitNeonCall()
4152 return Builder.CreateCall(F, Ops, name); in EmitNeonCall()
4975 SmallVectorImpl<Value *> &Ops, in EmitCommonNeonSISDBuiltinExpr() argument
4998 std::swap(Ops[0], Ops[1]); in EmitCommonNeonSISDBuiltinExpr()
5014 if (Ops[j]->getType()->getPrimitiveSizeInBits() == in EmitCommonNeonSISDBuiltinExpr()
5018 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy()); in EmitCommonNeonSISDBuiltinExpr()
5021 Ops[j] = in EmitCommonNeonSISDBuiltinExpr()
5022 CGF.Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType()); in EmitCommonNeonSISDBuiltinExpr()
5023 Ops[j] = in EmitCommonNeonSISDBuiltinExpr()
5024 CGF.Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0); in EmitCommonNeonSISDBuiltinExpr()
5027 Value *Result = CGF.EmitNeonCall(F, Ops, s); in EmitCommonNeonSISDBuiltinExpr()
5039 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1, in EmitCommonNeonBuiltinExpr() argument
5071 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs"); in EmitCommonNeonBuiltinExpr()
5072 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs"); in EmitCommonNeonBuiltinExpr()
5078 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
5079 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); in EmitCommonNeonBuiltinExpr()
5080 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn"); in EmitCommonNeonBuiltinExpr()
5085 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn"); in EmitCommonNeonBuiltinExpr()
5088 return Builder.CreateTrunc(Ops[0], VTy, "vaddhn"); in EmitCommonNeonBuiltinExpr()
5094 std::swap(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
5116 return EmitNeonCall(F, Ops, NameHint); in EmitCommonNeonBuiltinExpr()
5120 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ, in EmitCommonNeonBuiltinExpr()
5124 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE, in EmitCommonNeonBuiltinExpr()
5128 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE, in EmitCommonNeonBuiltinExpr()
5132 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT, in EmitCommonNeonBuiltinExpr()
5136 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT, in EmitCommonNeonBuiltinExpr()
5142 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef())); in EmitCommonNeonBuiltinExpr()
5146 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
5149 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") in EmitCommonNeonBuiltinExpr()
5150 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); in EmitCommonNeonBuiltinExpr()
5153 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
5156 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") in EmitCommonNeonBuiltinExpr()
5157 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); in EmitCommonNeonBuiltinExpr()
5167 return EmitNeonCall(F, Ops, "vcvt_n"); in EmitCommonNeonBuiltinExpr()
5183 return EmitNeonCall(F, Ops, "vcvt_n"); in EmitCommonNeonBuiltinExpr()
5197 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); in EmitCommonNeonBuiltinExpr()
5198 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt") in EmitCommonNeonBuiltinExpr()
5199 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt"); in EmitCommonNeonBuiltinExpr()
5250 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
5254 int CV = cast<ConstantInt>(Ops[2])->getSExtValue(); in EmitCommonNeonBuiltinExpr()
5259 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
5260 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
5261 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices, "vext"); in EmitCommonNeonBuiltinExpr()
5266 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
5267 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
5268 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
5271 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); in EmitCommonNeonBuiltinExpr()
5276 Ops.push_back(getAlignmentValue32(PtrOp0)); in EmitCommonNeonBuiltinExpr()
5277 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1"); in EmitCommonNeonBuiltinExpr()
5286 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitCommonNeonBuiltinExpr()
5289 Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN"); in EmitCommonNeonBuiltinExpr()
5290 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitCommonNeonBuiltinExpr()
5291 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
5292 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitCommonNeonBuiltinExpr()
5309 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint); in EmitCommonNeonBuiltinExpr()
5310 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitCommonNeonBuiltinExpr()
5311 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
5312 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitCommonNeonBuiltinExpr()
5321 Ops[0] = Builder.CreateInsertElement(V, Ld, CI); in EmitCommonNeonBuiltinExpr()
5322 return EmitNeonSplat(Ops[0], CI); in EmitCommonNeonBuiltinExpr()
5332 for (unsigned I = 2; I < Ops.size() - 1; ++I) in EmitCommonNeonBuiltinExpr()
5333 Ops[I] = Builder.CreateBitCast(Ops[I], Ty); in EmitCommonNeonBuiltinExpr()
5334 Ops.push_back(getAlignmentValue32(PtrOp1)); in EmitCommonNeonBuiltinExpr()
5335 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint); in EmitCommonNeonBuiltinExpr()
5336 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitCommonNeonBuiltinExpr()
5337 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
5338 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitCommonNeonBuiltinExpr()
5342 Ops[0] = Builder.CreateBitCast(Ops[0], DTy); in EmitCommonNeonBuiltinExpr()
5344 return Builder.CreateZExt(Ops[0], Ty, "vmovl"); in EmitCommonNeonBuiltinExpr()
5345 return Builder.CreateSExt(Ops[0], Ty, "vmovl"); in EmitCommonNeonBuiltinExpr()
5349 Ops[0] = Builder.CreateBitCast(Ops[0], QTy); in EmitCommonNeonBuiltinExpr()
5350 return Builder.CreateTrunc(Ops[0], Ty, "vmovn"); in EmitCommonNeonBuiltinExpr()
5360 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); in EmitCommonNeonBuiltinExpr()
5370 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
5380 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl"); in EmitCommonNeonBuiltinExpr()
5384 SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end()); in EmitCommonNeonBuiltinExpr()
5385 Ops[1] = in EmitCommonNeonBuiltinExpr()
5387 Ops.resize(2); in EmitCommonNeonBuiltinExpr()
5388 return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
5392 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n", in EmitCommonNeonBuiltinExpr()
5396 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n", in EmitCommonNeonBuiltinExpr()
5403 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
5407 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
5410 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", in EmitCommonNeonBuiltinExpr()
5414 Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false); in EmitCommonNeonBuiltinExpr()
5415 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1], in EmitCommonNeonBuiltinExpr()
5419 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
5421 Ops[0] = Builder.CreateZExt(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
5423 Ops[0] = Builder.CreateSExt(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
5424 Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false); in EmitCommonNeonBuiltinExpr()
5425 return Builder.CreateShl(Ops[0], Ops[1], "vshll_n"); in EmitCommonNeonBuiltinExpr()
5429 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
5430 Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false); in EmitCommonNeonBuiltinExpr()
5432 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
5434 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
5435 return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n"); in EmitCommonNeonBuiltinExpr()
5439 return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n"); in EmitCommonNeonBuiltinExpr()
5455 Ops.push_back(getAlignmentValue32(PtrOp0)); in EmitCommonNeonBuiltinExpr()
5456 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, ""); in EmitCommonNeonBuiltinExpr()
5469 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitCommonNeonBuiltinExpr()
5470 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, ""); in EmitCommonNeonBuiltinExpr()
5473 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, ""); in EmitCommonNeonBuiltinExpr()
5480 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
5481 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); in EmitCommonNeonBuiltinExpr()
5482 Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn"); in EmitCommonNeonBuiltinExpr()
5487 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn"); in EmitCommonNeonBuiltinExpr()
5490 return Builder.CreateTrunc(Ops[0], VTy, "vsubhn"); in EmitCommonNeonBuiltinExpr()
5494 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitCommonNeonBuiltinExpr()
5495 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
5496 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
5505 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
5506 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn"); in EmitCommonNeonBuiltinExpr()
5513 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
5514 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
5515 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
5516 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], in EmitCommonNeonBuiltinExpr()
5518 return Builder.CreateSExt(Ops[0], Ty, "vtst"); in EmitCommonNeonBuiltinExpr()
5522 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitCommonNeonBuiltinExpr()
5523 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
5524 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
5532 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
5533 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp"); in EmitCommonNeonBuiltinExpr()
5540 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitCommonNeonBuiltinExpr()
5541 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
5542 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
5551 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
5552 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip"); in EmitCommonNeonBuiltinExpr()
5563 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vdot"); in EmitCommonNeonBuiltinExpr()
5570 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlal_low"); in EmitCommonNeonBuiltinExpr()
5577 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlsl_low"); in EmitCommonNeonBuiltinExpr()
5584 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlal_high"); in EmitCommonNeonBuiltinExpr()
5591 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlsl_high"); in EmitCommonNeonBuiltinExpr()
5600 Value *Result = EmitNeonCall(F, Ops, NameHint); in EmitCommonNeonBuiltinExpr()
5628 static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops, in packTBLDVectorList() argument
5638 llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType()); in packTBLDVectorList()
5644 int PairPos = 0, End = Ops.size() - 1; in packTBLDVectorList()
5646 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], in packTBLDVectorList()
5647 Ops[PairPos+1], Indices, in packTBLDVectorList()
5656 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], in packTBLDVectorList()
5723 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) }; in EmitSpecialRegisterBuiltin() local
5724 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitSpecialRegisterBuiltin()
5882 Value *Ops[2]; in EmitARMBuiltinExpr() local
5884 Ops[i] = EmitScalarExpr(E->getArg(i)); in EmitARMBuiltinExpr()
5888 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); in EmitARMBuiltinExpr()
6173 SmallVector<Value*, 4> Ops; in EmitARMBuiltinExpr() local
6204 Ops.push_back(PtrOp0.getPointer()); in EmitARMBuiltinExpr()
6231 Ops.push_back(PtrOp1.getPointer()); in EmitARMBuiltinExpr()
6237 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitARMBuiltinExpr()
6244 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); in EmitARMBuiltinExpr()
6261 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane"); in EmitARMBuiltinExpr()
6279 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitARMBuiltinExpr()
6282 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops, in EmitARMBuiltinExpr()
6285 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops, in EmitARMBuiltinExpr()
6288 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops, in EmitARMBuiltinExpr()
6291 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops, in EmitARMBuiltinExpr()
6300 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0], in EmitARMBuiltinExpr()
6301 Ops[3], Ops[4], Ops[5]}); in EmitARMBuiltinExpr()
6464 return Builder.CreateCall(F, Ops, "vcvtr"); in EmitARMBuiltinExpr()
6486 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch); in EmitARMBuiltinExpr()
6496 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
6497 uint32_t Lane = cast<ConstantInt>(Ops[2])->getZExtValue(); in EmitARMBuiltinExpr()
6499 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); in EmitARMBuiltinExpr()
6505 Value *Ld = Builder.CreateCall(F, {Ops[0], Align}); in EmitARMBuiltinExpr()
6509 return Builder.CreateShuffleVector(Ops[1], Ld, SV, "vld1q_lane"); in EmitARMBuiltinExpr()
6513 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
6516 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane"); in EmitARMBuiltinExpr()
6521 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n", in EmitARMBuiltinExpr()
6525 Ops, "vqrshrun_n", 1, true); in EmitARMBuiltinExpr()
6528 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n", in EmitARMBuiltinExpr()
6532 Ops, "vqshrun_n", 1, true); in EmitARMBuiltinExpr()
6536 Ops, "vrecpe"); in EmitARMBuiltinExpr()
6539 Ops, "vrshrn_n", 1, true); in EmitARMBuiltinExpr()
6542 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitARMBuiltinExpr()
6543 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
6544 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true); in EmitARMBuiltinExpr()
6546 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]}); in EmitARMBuiltinExpr()
6547 return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n"); in EmitARMBuiltinExpr()
6554 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift); in EmitARMBuiltinExpr()
6556 Ops, "vsli_n"); in EmitARMBuiltinExpr()
6559 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitARMBuiltinExpr()
6560 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); in EmitARMBuiltinExpr()
6561 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitARMBuiltinExpr()
6566 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
6567 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2])); in EmitARMBuiltinExpr()
6568 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); in EmitARMBuiltinExpr()
6569 Ops[2] = getAlignmentValue32(PtrOp0); in EmitARMBuiltinExpr()
6570 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()}; in EmitARMBuiltinExpr()
6572 Tys), Ops); in EmitARMBuiltinExpr()
6576 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
6577 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); in EmitARMBuiltinExpr()
6578 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitARMBuiltinExpr()
6579 auto St = Builder.CreateStore(Ops[1], Builder.CreateBitCast(PtrOp0, Ty)); in EmitARMBuiltinExpr()
6584 Ops, "vtbl1"); in EmitARMBuiltinExpr()
6587 Ops, "vtbl2"); in EmitARMBuiltinExpr()
6590 Ops, "vtbl3"); in EmitARMBuiltinExpr()
6593 Ops, "vtbl4"); in EmitARMBuiltinExpr()
6596 Ops, "vtbx1"); in EmitARMBuiltinExpr()
6599 Ops, "vtbx2"); in EmitARMBuiltinExpr()
6602 Ops, "vtbx3"); in EmitARMBuiltinExpr()
6605 Ops, "vtbx4"); in EmitARMBuiltinExpr()
6611 SmallVectorImpl<Value *> &Ops, in EmitAArch64TblBuiltinExpr() argument
6667 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 1), nullptr, in EmitAArch64TblBuiltinExpr()
6668 Ops[1], Ty, Intrinsic::aarch64_neon_tbl1, in EmitAArch64TblBuiltinExpr()
6672 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 2), nullptr, in EmitAArch64TblBuiltinExpr()
6673 Ops[2], Ty, Intrinsic::aarch64_neon_tbl1, in EmitAArch64TblBuiltinExpr()
6677 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 3), nullptr, in EmitAArch64TblBuiltinExpr()
6678 Ops[3], Ty, Intrinsic::aarch64_neon_tbl2, in EmitAArch64TblBuiltinExpr()
6682 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 4), nullptr, in EmitAArch64TblBuiltinExpr()
6683 Ops[4], Ty, Intrinsic::aarch64_neon_tbl2, in EmitAArch64TblBuiltinExpr()
6688 packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 1), nullptr, Ops[2], in EmitAArch64TblBuiltinExpr()
6692 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV); in EmitAArch64TblBuiltinExpr()
6695 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); in EmitAArch64TblBuiltinExpr()
6700 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 2), Ops[0], in EmitAArch64TblBuiltinExpr()
6701 Ops[3], Ty, Intrinsic::aarch64_neon_tbx1, in EmitAArch64TblBuiltinExpr()
6706 packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 3), nullptr, Ops[4], in EmitAArch64TblBuiltinExpr()
6710 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4], in EmitAArch64TblBuiltinExpr()
6714 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); in EmitAArch64TblBuiltinExpr()
6719 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 4), Ops[0], in EmitAArch64TblBuiltinExpr()
6720 Ops[5], Ty, Intrinsic::aarch64_neon_tbx2, in EmitAArch64TblBuiltinExpr()
6754 return CGF.EmitNeonCall(F, Ops, s); in EmitAArch64TblBuiltinExpr()
6843 Value *Ops[2]; in EmitAArch64BuiltinExpr() local
6845 Ops[i] = EmitScalarExpr(E->getArg(i)); in EmitAArch64BuiltinExpr()
6849 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); in EmitAArch64BuiltinExpr()
6955 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)}; in EmitAArch64BuiltinExpr() local
6956 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitAArch64BuiltinExpr()
7050 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) }; in EmitAArch64BuiltinExpr() local
7051 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitAArch64BuiltinExpr()
7081 llvm::SmallVector<Value*, 4> Ops; in EmitAArch64BuiltinExpr() local
7084 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitAArch64BuiltinExpr()
7092 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); in EmitAArch64BuiltinExpr()
7101 Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1))); in EmitAArch64BuiltinExpr()
7102 Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E); in EmitAArch64BuiltinExpr()
7121 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
7122 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, HalfTy), Ops, "vabs"); in EmitAArch64BuiltinExpr()
7132 Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy); in EmitAArch64BuiltinExpr()
7141 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
7142 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; in EmitAArch64BuiltinExpr()
7145 Ops[0] = Builder.CreateBitCast(Ops[0], FTy); in EmitAArch64BuiltinExpr()
7147 return Builder.CreateFPToUI(Ops[0], InTy); in EmitAArch64BuiltinExpr()
7148 return Builder.CreateFPToSI(Ops[0], InTy); in EmitAArch64BuiltinExpr()
7156 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
7157 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; in EmitAArch64BuiltinExpr()
7160 Ops[0] = Builder.CreateBitCast(Ops[0], InTy); in EmitAArch64BuiltinExpr()
7162 return Builder.CreateUIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
7163 return Builder.CreateSIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
7173 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
7176 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64) in EmitAArch64BuiltinExpr()
7178 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32) in EmitAArch64BuiltinExpr()
7182 Ops[0] = Builder.CreateBitCast(Ops[0], InTy); in EmitAArch64BuiltinExpr()
7184 return Builder.CreateUIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
7185 return Builder.CreateSIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
7191 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
7192 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
7194 return Builder.CreateFPToUI(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
7195 return Builder.CreateFPToSI(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
7201 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
7202 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
7204 return Builder.CreateFPToUI(Ops[0], Int32Ty); in EmitAArch64BuiltinExpr()
7205 return Builder.CreateFPToSI(Ops[0], Int32Ty); in EmitAArch64BuiltinExpr()
7211 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
7212 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
7214 return Builder.CreateFPToUI(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
7215 return Builder.CreateFPToSI(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
7229 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
7249 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvt"); in EmitAArch64BuiltinExpr()
7250 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
7260 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7268 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]); break; in EmitAArch64BuiltinExpr()
7270 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]); break; in EmitAArch64BuiltinExpr()
7272 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "facg"); in EmitAArch64BuiltinExpr()
7273 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
7281 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7289 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n"); in EmitAArch64BuiltinExpr()
7290 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
7298 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7303 Ops[0] = Builder.CreateSExt(Ops[0], InTy, "sext"); in EmitAArch64BuiltinExpr()
7307 Ops[0] = Builder.CreateZExt(Ops[0], InTy); in EmitAArch64BuiltinExpr()
7310 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n"); in EmitAArch64BuiltinExpr()
7354 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
7356 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
7362 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
7364 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
7370 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
7372 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
7378 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
7380 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
7386 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
7388 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
7392 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
7393 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
7394 Ops[0] = in EmitAArch64BuiltinExpr()
7395 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty)); in EmitAArch64BuiltinExpr()
7396 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqzd"); in EmitAArch64BuiltinExpr()
7412 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7413 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
7414 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); in EmitAArch64BuiltinExpr()
7415 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
7416 return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd"); in EmitAArch64BuiltinExpr()
7432 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7433 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy); in EmitAArch64BuiltinExpr()
7434 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy); in EmitAArch64BuiltinExpr()
7435 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
7436 return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd"); in EmitAArch64BuiltinExpr()
7452 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7453 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
7454 Ops[1] = Builder.CreateBitCast(Ops[1], HalfTy); in EmitAArch64BuiltinExpr()
7455 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
7456 return Builder.CreateSExt(Ops[0], Int16Ty, "vcmpd"); in EmitAArch64BuiltinExpr()
7482 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7483 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
7484 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
7485 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
7486 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd"); in EmitAArch64BuiltinExpr()
7490 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7491 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
7492 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
7493 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
7494 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], in EmitAArch64BuiltinExpr()
7496 return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd"); in EmitAArch64BuiltinExpr()
7508 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
7509 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
7512 Ops[1] = Builder.CreateBitCast(Ops[1], in EmitAArch64BuiltinExpr()
7514 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
7515 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
7518 Ops[1] = Builder.CreateBitCast(Ops[1], in EmitAArch64BuiltinExpr()
7520 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
7521 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
7525 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 8)); in EmitAArch64BuiltinExpr()
7526 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
7530 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 16)); in EmitAArch64BuiltinExpr()
7531 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
7535 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 4)); in EmitAArch64BuiltinExpr()
7536 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
7540 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 8)); in EmitAArch64BuiltinExpr()
7541 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
7545 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 2)); in EmitAArch64BuiltinExpr()
7546 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
7549 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
7551 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
7555 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4)); in EmitAArch64BuiltinExpr()
7556 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
7560 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 1)); in EmitAArch64BuiltinExpr()
7561 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
7564 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
7566 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
7570 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2)); in EmitAArch64BuiltinExpr()
7571 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
7574 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
7576 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
7579 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
7581 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
7585 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
7587 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
7591 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
7593 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
7596 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7597 return Builder.CreateFAdd(Ops[0], Ops[1], "vaddh"); in EmitAArch64BuiltinExpr()
7599 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7600 return Builder.CreateFSub(Ops[0], Ops[1], "vsubh"); in EmitAArch64BuiltinExpr()
7602 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7603 return Builder.CreateFMul(Ops[0], Ops[1], "vmulh"); in EmitAArch64BuiltinExpr()
7605 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7606 return Builder.CreateFDiv(Ops[0], Ops[1], "vdivh"); in EmitAArch64BuiltinExpr()
7611 {EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2)), Ops[0]}); in EmitAArch64BuiltinExpr()
7618 return Builder.CreateCall(F, {Sub, EmitScalarExpr(E->getArg(2)), Ops[0]}); in EmitAArch64BuiltinExpr()
7622 return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd"); in EmitAArch64BuiltinExpr()
7625 return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd"); in EmitAArch64BuiltinExpr()
7629 ProductOps.push_back(vectorWrapScalar16(Ops[1])); in EmitAArch64BuiltinExpr()
7632 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), in EmitAArch64BuiltinExpr()
7635 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); in EmitAArch64BuiltinExpr()
7640 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
7643 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7644 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
7646 Ops, "vqshlu_n"); in EmitAArch64BuiltinExpr()
7653 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7654 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
7655 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n"); in EmitAArch64BuiltinExpr()
7662 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7663 int SV = cast<ConstantInt>(Ops[1])->getSExtValue(); in EmitAArch64BuiltinExpr()
7664 Ops[1] = ConstantInt::get(Int64Ty, -SV); in EmitAArch64BuiltinExpr()
7665 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n"); in EmitAArch64BuiltinExpr()
7672 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
7673 Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2)))); in EmitAArch64BuiltinExpr()
7674 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty), in EmitAArch64BuiltinExpr()
7675 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)}); in EmitAArch64BuiltinExpr()
7676 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty)); in EmitAArch64BuiltinExpr()
7682 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n"); in EmitAArch64BuiltinExpr()
7687 Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), in EmitAArch64BuiltinExpr()
7697 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt), in EmitAArch64BuiltinExpr()
7702 Ops[1] = Builder.CreateAShr( in EmitAArch64BuiltinExpr()
7703 Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), in EmitAArch64BuiltinExpr()
7706 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
7714 return Ops[0]; in EmitAArch64BuiltinExpr()
7715 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt), in EmitAArch64BuiltinExpr()
7717 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
7723 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), in EmitAArch64BuiltinExpr()
7726 ProductOps.push_back(vectorWrapScalar16(Ops[1])); in EmitAArch64BuiltinExpr()
7727 ProductOps.push_back(vectorWrapScalar16(Ops[2])); in EmitAArch64BuiltinExpr()
7729 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), in EmitAArch64BuiltinExpr()
7732 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); in EmitAArch64BuiltinExpr()
7733 Ops.pop_back(); in EmitAArch64BuiltinExpr()
7739 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
7744 ProductOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
7746 Ops[1] = in EmitAArch64BuiltinExpr()
7753 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
7759 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), in EmitAArch64BuiltinExpr()
7762 ProductOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
7763 ProductOps.push_back(Ops[2]); in EmitAArch64BuiltinExpr()
7764 Ops[1] = in EmitAArch64BuiltinExpr()
7767 Ops.pop_back(); in EmitAArch64BuiltinExpr()
7773 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
7790 Builtin->NameHint, Builtin->TypeModifier, E, Ops, in EmitAArch64BuiltinExpr()
7793 if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops, Arch)) in EmitAArch64BuiltinExpr()
7802 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
7803 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
7804 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
7806 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl"); in EmitAArch64BuiltinExpr()
7807 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl"); in EmitAArch64BuiltinExpr()
7808 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl"); in EmitAArch64BuiltinExpr()
7809 return Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
7815 Value *Addend = Ops[0]; in EmitAArch64BuiltinExpr()
7816 Value *Multiplicand = Ops[1]; in EmitAArch64BuiltinExpr()
7817 Value *LaneSource = Ops[2]; in EmitAArch64BuiltinExpr()
7818 Ops[0] = Multiplicand; in EmitAArch64BuiltinExpr()
7819 Ops[1] = LaneSource; in EmitAArch64BuiltinExpr()
7820 Ops[2] = Addend; in EmitAArch64BuiltinExpr()
7826 llvm::Constant *cst = cast<Constant>(Ops[3]); in EmitAArch64BuiltinExpr()
7828 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy); in EmitAArch64BuiltinExpr()
7829 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane"); in EmitAArch64BuiltinExpr()
7831 Ops.pop_back(); in EmitAArch64BuiltinExpr()
7833 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla"); in EmitAArch64BuiltinExpr()
7839 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
7840 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); in EmitAArch64BuiltinExpr()
7843 Ops[2] = Builder.CreateBitCast(Ops[2], VTy); in EmitAArch64BuiltinExpr()
7844 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); in EmitAArch64BuiltinExpr()
7846 Value *Result = Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); in EmitAArch64BuiltinExpr()
7850 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
7851 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
7855 Ops[2] = Builder.CreateBitCast(Ops[2], STy); in EmitAArch64BuiltinExpr()
7857 cast<ConstantInt>(Ops[3])); in EmitAArch64BuiltinExpr()
7858 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane"); in EmitAArch64BuiltinExpr()
7860 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]}); in EmitAArch64BuiltinExpr()
7864 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
7865 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
7867 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
7868 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3])); in EmitAArch64BuiltinExpr()
7869 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]}); in EmitAArch64BuiltinExpr()
7877 Ops.push_back(EmitScalarExpr(E->getArg(3))); in EmitAArch64BuiltinExpr()
7880 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); in EmitAArch64BuiltinExpr()
7881 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); in EmitAArch64BuiltinExpr()
7887 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); in EmitAArch64BuiltinExpr()
7893 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax"); in EmitAArch64BuiltinExpr()
7895 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7897 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmax"); in EmitAArch64BuiltinExpr()
7904 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin"); in EmitAArch64BuiltinExpr()
7906 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7908 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmin"); in EmitAArch64BuiltinExpr()
7915 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd"); in EmitAArch64BuiltinExpr()
7926 TmpOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
7929 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType()); in EmitAArch64BuiltinExpr()
7937 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin"); in EmitAArch64BuiltinExpr()
7943 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax"); in EmitAArch64BuiltinExpr()
7947 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm"); in EmitAArch64BuiltinExpr()
7949 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7951 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vminnm"); in EmitAArch64BuiltinExpr()
7955 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm"); in EmitAArch64BuiltinExpr()
7957 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7959 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmaxnm"); in EmitAArch64BuiltinExpr()
7961 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7963 Ops, "vrecps"); in EmitAArch64BuiltinExpr()
7966 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7968 Ops, "vrecps"); in EmitAArch64BuiltinExpr()
7970 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
7972 Ops, "vrecps"); in EmitAArch64BuiltinExpr()
7975 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n"); in EmitAArch64BuiltinExpr()
7978 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n"); in EmitAArch64BuiltinExpr()
7981 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n"); in EmitAArch64BuiltinExpr()
7984 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n"); in EmitAArch64BuiltinExpr()
7987 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n"); in EmitAArch64BuiltinExpr()
7989 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
7991 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrnda"); in EmitAArch64BuiltinExpr()
7996 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda"); in EmitAArch64BuiltinExpr()
7999 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8001 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndi"); in EmitAArch64BuiltinExpr()
8004 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8006 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndm"); in EmitAArch64BuiltinExpr()
8011 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm"); in EmitAArch64BuiltinExpr()
8014 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8016 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndn"); in EmitAArch64BuiltinExpr()
8021 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn"); in EmitAArch64BuiltinExpr()
8024 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8026 return EmitNeonCall(CGM.getIntrinsic(Int, FloatTy), Ops, "vrndn"); in EmitAArch64BuiltinExpr()
8029 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8031 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndp"); in EmitAArch64BuiltinExpr()
8036 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp"); in EmitAArch64BuiltinExpr()
8039 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8041 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndx"); in EmitAArch64BuiltinExpr()
8046 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx"); in EmitAArch64BuiltinExpr()
8049 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8051 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndz"); in EmitAArch64BuiltinExpr()
8056 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz"); in EmitAArch64BuiltinExpr()
8060 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
8062 return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") in EmitAArch64BuiltinExpr()
8063 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
8068 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); in EmitAArch64BuiltinExpr()
8070 return Builder.CreateFPExt(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
8076 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); in EmitAArch64BuiltinExpr()
8078 return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
8092 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); in EmitAArch64BuiltinExpr()
8094 return Builder.CreateFPToUI(Ops[0], Ty); in EmitAArch64BuiltinExpr()
8095 return Builder.CreateFPToSI(Ops[0], Ty); in EmitAArch64BuiltinExpr()
8111 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta"); in EmitAArch64BuiltinExpr()
8127 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm"); in EmitAArch64BuiltinExpr()
8143 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn"); in EmitAArch64BuiltinExpr()
8159 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp"); in EmitAArch64BuiltinExpr()
8164 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx"); in EmitAArch64BuiltinExpr()
8170 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
8171 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract"); in EmitAArch64BuiltinExpr()
8172 Ops.pop_back(); in EmitAArch64BuiltinExpr()
8174 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmulx"); in EmitAArch64BuiltinExpr()
8182 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
8185 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); in EmitAArch64BuiltinExpr()
8186 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract"); in EmitAArch64BuiltinExpr()
8187 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
8197 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm"); in EmitAArch64BuiltinExpr()
8202 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm"); in EmitAArch64BuiltinExpr()
8205 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8207 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vsqrt"); in EmitAArch64BuiltinExpr()
8212 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
8213 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt"); in EmitAArch64BuiltinExpr()
8218 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit"); in EmitAArch64BuiltinExpr()
8229 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8230 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
8231 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
8241 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8242 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
8243 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
8253 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8254 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
8255 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
8265 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8266 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
8267 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
8274 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8275 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
8276 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
8283 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8284 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
8285 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
8292 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8293 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
8294 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
8301 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8302 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
8303 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
8310 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8311 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
8312 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
8319 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8320 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
8321 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
8328 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8329 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
8330 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
8337 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8338 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
8339 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
8346 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8347 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
8348 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
8355 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8356 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
8357 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
8364 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8365 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
8366 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
8373 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8374 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
8375 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
8382 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8383 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
8384 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
8391 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8392 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
8393 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
8400 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8401 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
8402 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
8409 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8410 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
8411 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
8418 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8419 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
8420 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
8427 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8428 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
8429 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
8436 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8437 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
8438 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
8445 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8446 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
8447 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
8454 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8455 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv"); in EmitAArch64BuiltinExpr()
8456 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
8463 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8464 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv"); in EmitAArch64BuiltinExpr()
8465 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
8472 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8473 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv"); in EmitAArch64BuiltinExpr()
8474 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
8481 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8482 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv"); in EmitAArch64BuiltinExpr()
8483 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
8486 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
8488 return Builder.CreateFMul(Ops[0], RHS); in EmitAArch64BuiltinExpr()
8495 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8496 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
8497 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
8504 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8505 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
8512 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8513 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
8514 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
8521 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8522 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
8529 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8530 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
8531 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
8538 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8539 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
8546 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8547 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
8548 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
8555 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
8556 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
8562 return EmitNeonCall(Intrin, Ops, "vsri_n"); in EmitAArch64BuiltinExpr()
8568 return EmitNeonCall(Intrin, Ops, "vsli_n"); in EmitAArch64BuiltinExpr()
8572 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
8573 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); in EmitAArch64BuiltinExpr()
8574 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
8579 TmpOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
8580 TmpOps.push_back(Ops[2]); in EmitAArch64BuiltinExpr()
8583 Ops[0] = Builder.CreateBitCast(Ops[0], VTy); in EmitAArch64BuiltinExpr()
8584 return Builder.CreateAdd(Ops[0], tmp); in EmitAArch64BuiltinExpr()
8588 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); in EmitAArch64BuiltinExpr()
8591 return Builder.CreateAlignedLoad(VTy, Ops[0], Alignment); in EmitAArch64BuiltinExpr()
8595 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); in EmitAArch64BuiltinExpr()
8596 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); in EmitAArch64BuiltinExpr()
8597 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
8600 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
8602 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
8605 Ops[0] = in EmitAArch64BuiltinExpr()
8606 Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], Alignment); in EmitAArch64BuiltinExpr()
8607 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane"); in EmitAArch64BuiltinExpr()
8613 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
8616 Ops[0] = in EmitAArch64BuiltinExpr()
8617 Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], Alignment); in EmitAArch64BuiltinExpr()
8619 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI); in EmitAArch64BuiltinExpr()
8620 return EmitNeonSplat(Ops[0], CI); in EmitAArch64BuiltinExpr()
8624 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
8625 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); in EmitAArch64BuiltinExpr()
8626 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
8627 return Builder.CreateDefaultAlignedStore(Ops[1], in EmitAArch64BuiltinExpr()
8628 Builder.CreateBitCast(Ops[0], Ty)); in EmitAArch64BuiltinExpr()
8632 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
8635 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); in EmitAArch64BuiltinExpr()
8636 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
8637 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
8638 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
8643 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
8646 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); in EmitAArch64BuiltinExpr()
8647 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
8648 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
8649 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
8654 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
8657 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); in EmitAArch64BuiltinExpr()
8658 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
8659 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
8660 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
8666 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
8669 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); in EmitAArch64BuiltinExpr()
8670 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
8671 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
8672 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
8678 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
8681 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); in EmitAArch64BuiltinExpr()
8682 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
8683 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
8684 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
8690 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
8693 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); in EmitAArch64BuiltinExpr()
8694 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
8695 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
8696 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
8700 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
8702 Ops.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
8703 Ops.erase(Ops.begin()+1); in EmitAArch64BuiltinExpr()
8704 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
8705 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
8706 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); in EmitAArch64BuiltinExpr()
8707 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane"); in EmitAArch64BuiltinExpr()
8708 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
8709 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
8710 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
8714 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
8716 Ops.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
8717 Ops.erase(Ops.begin()+1); in EmitAArch64BuiltinExpr()
8718 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
8719 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
8720 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); in EmitAArch64BuiltinExpr()
8721 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); in EmitAArch64BuiltinExpr()
8722 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane"); in EmitAArch64BuiltinExpr()
8723 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
8724 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
8725 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
8729 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
8731 Ops.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
8732 Ops.erase(Ops.begin()+1); in EmitAArch64BuiltinExpr()
8733 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
8734 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
8735 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); in EmitAArch64BuiltinExpr()
8736 Ops[4] = Builder.CreateBitCast(Ops[4], Ty); in EmitAArch64BuiltinExpr()
8737 Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty); in EmitAArch64BuiltinExpr()
8738 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld4_lane"); in EmitAArch64BuiltinExpr()
8739 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
8740 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
8741 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
8745 Ops.push_back(Ops[0]); in EmitAArch64BuiltinExpr()
8746 Ops.erase(Ops.begin()); in EmitAArch64BuiltinExpr()
8747 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() }; in EmitAArch64BuiltinExpr()
8749 Ops, ""); in EmitAArch64BuiltinExpr()
8753 Ops.push_back(Ops[0]); in EmitAArch64BuiltinExpr()
8754 Ops.erase(Ops.begin()); in EmitAArch64BuiltinExpr()
8755 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty); in EmitAArch64BuiltinExpr()
8756 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; in EmitAArch64BuiltinExpr()
8758 Ops, ""); in EmitAArch64BuiltinExpr()
8762 Ops.push_back(Ops[0]); in EmitAArch64BuiltinExpr()
8763 Ops.erase(Ops.begin()); in EmitAArch64BuiltinExpr()
8764 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; in EmitAArch64BuiltinExpr()
8766 Ops, ""); in EmitAArch64BuiltinExpr()
8770 Ops.push_back(Ops[0]); in EmitAArch64BuiltinExpr()
8771 Ops.erase(Ops.begin()); in EmitAArch64BuiltinExpr()
8772 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); in EmitAArch64BuiltinExpr()
8773 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; in EmitAArch64BuiltinExpr()
8775 Ops, ""); in EmitAArch64BuiltinExpr()
8779 Ops.push_back(Ops[0]); in EmitAArch64BuiltinExpr()
8780 Ops.erase(Ops.begin()); in EmitAArch64BuiltinExpr()
8781 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; in EmitAArch64BuiltinExpr()
8783 Ops, ""); in EmitAArch64BuiltinExpr()
8787 Ops.push_back(Ops[0]); in EmitAArch64BuiltinExpr()
8788 Ops.erase(Ops.begin()); in EmitAArch64BuiltinExpr()
8789 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); in EmitAArch64BuiltinExpr()
8790 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() }; in EmitAArch64BuiltinExpr()
8792 Ops, ""); in EmitAArch64BuiltinExpr()
8796 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitAArch64BuiltinExpr()
8797 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
8798 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
8807 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
8808 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn"); in EmitAArch64BuiltinExpr()
8815 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitAArch64BuiltinExpr()
8816 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
8817 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
8825 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
8826 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp"); in EmitAArch64BuiltinExpr()
8833 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitAArch64BuiltinExpr()
8834 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
8835 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
8844 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
8845 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip"); in EmitAArch64BuiltinExpr()
8852 Ops, "vtbl1"); in EmitAArch64BuiltinExpr()
8856 Ops, "vtbl2"); in EmitAArch64BuiltinExpr()
8860 Ops, "vtbl3"); in EmitAArch64BuiltinExpr()
8864 Ops, "vtbl4"); in EmitAArch64BuiltinExpr()
8868 Ops, "vtbx1"); in EmitAArch64BuiltinExpr()
8872 Ops, "vtbx2"); in EmitAArch64BuiltinExpr()
8876 Ops, "vtbx3"); in EmitAArch64BuiltinExpr()
8880 Ops, "vtbx4"); in EmitAArch64BuiltinExpr()
8885 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd"); in EmitAArch64BuiltinExpr()
8890 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd"); in EmitAArch64BuiltinExpr()
9051 BuildVector(ArrayRef<llvm::Value*> Ops) { in BuildVector() argument
9052 assert((Ops.size() & (Ops.size() - 1)) == 0 && in BuildVector()
9055 for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i) in BuildVector()
9056 AllConstants &= isa<Constant>(Ops[i]); in BuildVector()
9061 for (unsigned i = 0, e = Ops.size(); i != e; ++i) in BuildVector()
9062 CstOps.push_back(cast<Constant>(Ops[i])); in BuildVector()
9068 llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size())); in BuildVector()
9070 for (unsigned i = 0, e = Ops.size(); i != e; ++i) in BuildVector()
9071 Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i)); in BuildVector()
9098 ArrayRef<Value *> Ops, in EmitX86MaskedStore() argument
9101 Value *Ptr = CGF.Builder.CreateBitCast(Ops[0], in EmitX86MaskedStore()
9102 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitX86MaskedStore()
9104 Value *MaskVec = getMaskVecValue(CGF, Ops[2], in EmitX86MaskedStore()
9105 Ops[1]->getType()->getVectorNumElements()); in EmitX86MaskedStore()
9107 return CGF.Builder.CreateMaskedStore(Ops[1], Ptr, Align, MaskVec); in EmitX86MaskedStore()
9111 ArrayRef<Value *> Ops, unsigned Align) { in EmitX86MaskedLoad() argument
9113 Value *Ptr = CGF.Builder.CreateBitCast(Ops[0], in EmitX86MaskedLoad()
9114 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitX86MaskedLoad()
9116 Value *MaskVec = getMaskVecValue(CGF, Ops[2], in EmitX86MaskedLoad()
9117 Ops[1]->getType()->getVectorNumElements()); in EmitX86MaskedLoad()
9119 return CGF.Builder.CreateMaskedLoad(Ptr, Align, MaskVec, Ops[1]); in EmitX86MaskedLoad()
9123 ArrayRef<Value *> Ops) { in EmitX86ExpandLoad() argument
9124 llvm::Type *ResultTy = Ops[1]->getType(); in EmitX86ExpandLoad()
9128 Value *Ptr = CGF.Builder.CreateBitCast(Ops[0], in EmitX86ExpandLoad()
9131 Value *MaskVec = getMaskVecValue(CGF, Ops[2], in EmitX86ExpandLoad()
9136 return CGF.Builder.CreateCall(F, { Ptr, MaskVec, Ops[1] }); in EmitX86ExpandLoad()
9140 ArrayRef<Value *> Ops) { in EmitX86CompressStore() argument
9141 llvm::Type *ResultTy = Ops[1]->getType(); in EmitX86CompressStore()
9145 Value *Ptr = CGF.Builder.CreateBitCast(Ops[0], in EmitX86CompressStore()
9148 Value *MaskVec = getMaskVecValue(CGF, Ops[2], in EmitX86CompressStore()
9153 return CGF.Builder.CreateCall(F, { Ops[1], Ptr, MaskVec }); in EmitX86CompressStore()
9157 ArrayRef<Value *> Ops, in EmitX86MaskLogic() argument
9159 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86MaskLogic()
9160 Value *LHS = getMaskVecValue(CGF, Ops[0], NumElts); in EmitX86MaskLogic()
9161 Value *RHS = getMaskVecValue(CGF, Ops[1], NumElts); in EmitX86MaskLogic()
9167 Ops[0]->getType()); in EmitX86MaskLogic()
9240 bool Signed, ArrayRef<Value *> Ops) { in EmitX86MaskedCompare() argument
9241 assert((Ops.size() == 2 || Ops.size() == 4) && in EmitX86MaskedCompare()
9243 unsigned NumElts = Ops[0]->getType()->getVectorNumElements(); in EmitX86MaskedCompare()
9263 Cmp = CGF.Builder.CreateICmp(Pred, Ops[0], Ops[1]); in EmitX86MaskedCompare()
9267 if (Ops.size() == 4) in EmitX86MaskedCompare()
9268 MaskIn = Ops[3]; in EmitX86MaskedCompare()
9278 static Value *EmitX86Abs(CodeGenFunction &CGF, ArrayRef<Value *> Ops) { in EmitX86Abs() argument
9280 llvm::Type *Ty = Ops[0]->getType(); in EmitX86Abs()
9282 Value *Sub = CGF.Builder.CreateSub(Zero, Ops[0]); in EmitX86Abs()
9283 Value *Cmp = CGF.Builder.CreateICmp(ICmpInst::ICMP_SGT, Ops[0], Zero); in EmitX86Abs()
9284 Value *Res = CGF.Builder.CreateSelect(Cmp, Ops[0], Sub); in EmitX86Abs()
9289 ArrayRef<Value *> Ops) { in EmitX86MinMax() argument
9290 Value *Cmp = CGF.Builder.CreateICmp(Pred, Ops[0], Ops[1]); in EmitX86MinMax()
9291 Value *Res = CGF.Builder.CreateSelect(Cmp, Ops[0], Ops[1]); in EmitX86MinMax()
9293 assert(Ops.size() == 2); in EmitX86MinMax()
9298 static Value *EmitX86FMAExpr(CodeGenFunction &CGF, ArrayRef<Value *> Ops, in EmitX86FMAExpr() argument
9337 Value *A = Ops[0]; in EmitX86FMAExpr()
9338 Value *B = Ops[1]; in EmitX86FMAExpr()
9339 Value *C = Ops[2]; in EmitX86FMAExpr()
9348 cast<llvm::ConstantInt>(Ops.back())->getZExtValue() != (uint64_t)4) { in EmitX86FMAExpr()
9350 Res = CGF.Builder.CreateCall(Intr, {A, B, C, Ops.back() }); in EmitX86FMAExpr()
9376 MaskFalseVal = Ops[0]; in EmitX86FMAExpr()
9382 MaskFalseVal = Constant::getNullValue(Ops[0]->getType()); in EmitX86FMAExpr()
9392 MaskFalseVal = Ops[2]; in EmitX86FMAExpr()
9397 return EmitX86Select(CGF, Ops[3], Res, MaskFalseVal); in EmitX86FMAExpr()
9403 EmitScalarFMAExpr(CodeGenFunction &CGF, MutableArrayRef<Value *> Ops, in EmitScalarFMAExpr() argument
9407 if (Ops.size() > 4) in EmitScalarFMAExpr()
9408 Rnd = cast<llvm::ConstantInt>(Ops[4])->getZExtValue(); in EmitScalarFMAExpr()
9411 Ops[2] = CGF.Builder.CreateFNeg(Ops[2]); in EmitScalarFMAExpr()
9413 Ops[0] = CGF.Builder.CreateExtractElement(Ops[0], (uint64_t)0); in EmitScalarFMAExpr()
9414 Ops[1] = CGF.Builder.CreateExtractElement(Ops[1], (uint64_t)0); in EmitScalarFMAExpr()
9415 Ops[2] = CGF.Builder.CreateExtractElement(Ops[2], (uint64_t)0); in EmitScalarFMAExpr()
9418 Intrinsic::ID IID = Ops[0]->getType()->getPrimitiveSizeInBits() == 32 ? in EmitScalarFMAExpr()
9422 {Ops[0], Ops[1], Ops[2], Ops[4]}); in EmitScalarFMAExpr()
9424 Function *FMA = CGF.CGM.getIntrinsic(Intrinsic::fma, Ops[0]->getType()); in EmitScalarFMAExpr()
9425 Res = CGF.Builder.CreateCall(FMA, Ops.slice(0, 3)); in EmitScalarFMAExpr()
9428 if (Ops.size() > 3) { in EmitScalarFMAExpr()
9430 : Ops[PTIdx]; in EmitScalarFMAExpr()
9438 Res = EmitX86ScalarSelect(CGF, Ops[3], Res, PassThru); in EmitScalarFMAExpr()
9444 ArrayRef<Value *> Ops) { in EmitX86Muldq() argument
9445 llvm::Type *Ty = Ops[0]->getType(); in EmitX86Muldq()
9449 Value *LHS = CGF.Builder.CreateBitCast(Ops[0], Ty); in EmitX86Muldq()
9450 Value *RHS = CGF.Builder.CreateBitCast(Ops[1], Ty); in EmitX86Muldq()
9473 ArrayRef<Value *> Ops) { in EmitX86Ternlog() argument
9474 llvm::Type *Ty = Ops[0]->getType(); in EmitX86Ternlog()
9495 Ops.drop_back()); in EmitX86Ternlog()
9496 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) : Ops[0]; in EmitX86Ternlog()
9497 return EmitX86Select(CGF, Ops[4], Ternlog, PassThru); in EmitX86Ternlog()
9509 ArrayRef<Value *> Ops, bool IsSigned, in EmitX86AddSubSatExpr() argument
9514 llvm::Function *F = CGF.CGM.getIntrinsic(IID, Ops[0]->getType()); in EmitX86AddSubSatExpr()
9515 return CGF.Builder.CreateCall(F, {Ops[0], Ops[1]}); in EmitX86AddSubSatExpr()
9666 SmallVector<Value*, 4> Ops; in EmitX86BuiltinExpr() local
9677 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitX86BuiltinExpr()
9686 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); in EmitX86BuiltinExpr()
9695 auto getCmpIntrinsicCall = [this, &Ops](Intrinsic::ID ID, unsigned Imm) { in EmitX86BuiltinExpr()
9696 Ops.push_back(llvm::ConstantInt::get(Int8Ty, Imm)); in EmitX86BuiltinExpr()
9698 return Builder.CreateCall(F, Ops); in EmitX86BuiltinExpr()
9706 auto getVectorFCmpIR = [this, &Ops](CmpInst::Predicate Pred) { in EmitX86BuiltinExpr()
9707 Value *Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
9708 llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
9717 Value *Address = Ops[0]; in EmitX86BuiltinExpr()
9718 ConstantInt *C = cast<ConstantInt>(Ops[1]); in EmitX86BuiltinExpr()
9727 Ops[0]); in EmitX86BuiltinExpr()
9747 Ops[0]); in EmitX86BuiltinExpr()
9753 Value *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType()); in EmitX86BuiltinExpr()
9754 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)}); in EmitX86BuiltinExpr()
9759 Value *F = CGM.getIntrinsic(Intrinsic::cttz, Ops[0]->getType()); in EmitX86BuiltinExpr()
9760 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)}); in EmitX86BuiltinExpr()
9774 return Builder.CreateBitCast(BuildVector(Ops), in EmitX86BuiltinExpr()
9786 unsigned NumElts = Ops[0]->getType()->getVectorNumElements(); in EmitX86BuiltinExpr()
9787 uint64_t Index = cast<ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
9791 return Builder.CreateExtractElement(Ops[0], Index); in EmitX86BuiltinExpr()
9801 unsigned NumElts = Ops[0]->getType()->getVectorNumElements(); in EmitX86BuiltinExpr()
9802 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
9806 return Builder.CreateInsertElement(Ops[0], Ops[1], Index); in EmitX86BuiltinExpr()
9811 Builder.CreateStore(Ops[0], Tmp); in EmitX86BuiltinExpr()
9856 Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty); in EmitX86BuiltinExpr()
9857 Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty); in EmitX86BuiltinExpr()
9858 Ops[1] = Mhi; in EmitX86BuiltinExpr()
9859 Ops.push_back(Mlo); in EmitX86BuiltinExpr()
9860 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
9880 return EmitX86MaskedStore(*this, Ops, 1); in EmitX86BuiltinExpr()
9884 return EmitX86MaskedStore(*this, Ops, 1); in EmitX86BuiltinExpr()
9900 return Builder.CreateCall(F, Ops); in EmitX86BuiltinExpr()
9914 return EmitX86SExtMask(*this, Ops[0], ConvertType(E->getType())); in EmitX86BuiltinExpr()
9928 return EmitX86ConvertToMask(*this, Ops[0]); in EmitX86BuiltinExpr()
9934 return EmitScalarFMAExpr(*this, Ops, Ops[0]); in EmitX86BuiltinExpr()
9937 return EmitScalarFMAExpr(*this, Ops, in EmitX86BuiltinExpr()
9938 Constant::getNullValue(Ops[0]->getType())); in EmitX86BuiltinExpr()
9941 return EmitScalarFMAExpr(*this, Ops, Ops[0], /*ZeroMask*/true); in EmitX86BuiltinExpr()
9944 return EmitScalarFMAExpr(*this, Ops, Ops[2], /*ZeroMask*/false, 2); in EmitX86BuiltinExpr()
9947 return EmitScalarFMAExpr(*this, Ops, Ops[2], /*ZeroMask*/false, 2, in EmitX86BuiltinExpr()
9961 return EmitX86FMAExpr(*this, Ops, BuiltinID, /*IsAddSub*/false); in EmitX86BuiltinExpr()
9974 return EmitX86FMAExpr(*this, Ops, BuiltinID, /*IsAddSub*/true); in EmitX86BuiltinExpr()
9990 return EmitX86MaskedStore(*this, Ops, Align); in EmitX86BuiltinExpr()
10010 return EmitX86MaskedLoad(*this, Ops, 1); in EmitX86BuiltinExpr()
10014 return EmitX86MaskedLoad(*this, Ops, 1); in EmitX86BuiltinExpr()
10030 return EmitX86MaskedLoad(*this, Ops, Align); in EmitX86BuiltinExpr()
10051 return EmitX86ExpandLoad(*this, Ops); in EmitX86BuiltinExpr()
10071 return EmitX86CompressStore(*this, Ops); in EmitX86BuiltinExpr()
10079 Ops[1] = Builder.CreateBitCast(Ops[1], VecTy, "cast"); in EmitX86BuiltinExpr()
10083 Ops[1] = Builder.CreateExtractElement(Ops[1], Index, "extract"); in EmitX86BuiltinExpr()
10086 Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy); in EmitX86BuiltinExpr()
10087 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitX86BuiltinExpr()
10107 unsigned SrcNumElts = Ops[0]->getType()->getVectorNumElements(); in EmitX86BuiltinExpr()
10109 unsigned Index = cast<ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
10118 Value *Res = Builder.CreateShuffleVector(Ops[0], in EmitX86BuiltinExpr()
10119 UndefValue::get(Ops[0]->getType()), in EmitX86BuiltinExpr()
10123 if (Ops.size() == 4) in EmitX86BuiltinExpr()
10124 Res = EmitX86Select(*this, Ops[3], Res, Ops[2]); in EmitX86BuiltinExpr()
10144 unsigned DstNumElts = Ops[0]->getType()->getVectorNumElements(); in EmitX86BuiltinExpr()
10145 unsigned SrcNumElts = Ops[1]->getType()->getVectorNumElements(); in EmitX86BuiltinExpr()
10147 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
10156 Value *Op1 = Builder.CreateShuffleVector(Ops[1], in EmitX86BuiltinExpr()
10157 UndefValue::get(Ops[1]->getType()), in EmitX86BuiltinExpr()
10168 return Builder.CreateShuffleVector(Ops[0], Op1, in EmitX86BuiltinExpr()
10174 Value *Res = Builder.CreateTrunc(Ops[0], Ops[1]->getType()); in EmitX86BuiltinExpr()
10175 return EmitX86Select(*this, Ops[2], Res, Ops[1]); in EmitX86BuiltinExpr()
10180 if (const auto *C = dyn_cast<Constant>(Ops[2])) in EmitX86BuiltinExpr()
10182 return Builder.CreateTrunc(Ops[0], Ops[1]->getType()); in EmitX86BuiltinExpr()
10199 return Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
10209 unsigned NumElts = Ops[0]->getType()->getVectorNumElements(); in EmitX86BuiltinExpr()
10210 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
10218 return Builder.CreateShuffleVector(Ops[0], Ops[1], in EmitX86BuiltinExpr()
10225 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
10226 llvm::Type *Ty = Ops[0]->getType(); in EmitX86BuiltinExpr()
10242 return Builder.CreateShuffleVector(Ops[0], UndefValue::get(Ty), in EmitX86BuiltinExpr()
10249 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
10250 llvm::Type *Ty = Ops[0]->getType(); in EmitX86BuiltinExpr()
10266 return Builder.CreateShuffleVector(Ops[0], UndefValue::get(Ty), in EmitX86BuiltinExpr()
10279 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
10280 llvm::Type *Ty = Ops[0]->getType(); in EmitX86BuiltinExpr()
10296 return Builder.CreateShuffleVector(Ops[0], UndefValue::get(Ty), in EmitX86BuiltinExpr()
10306 uint32_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
10307 llvm::Type *Ty = Ops[0]->getType(); in EmitX86BuiltinExpr()
10326 return Builder.CreateShuffleVector(Ops[0], Ops[1], in EmitX86BuiltinExpr()
10334 unsigned Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
10335 llvm::Type *Ty = Ops[0]->getType(); in EmitX86BuiltinExpr()
10344 return Builder.CreateShuffleVector(Ops[0], UndefValue::get(Ty), in EmitX86BuiltinExpr()
10351 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
10353 unsigned NumElts = Ops[0]->getType()->getVectorNumElements(); in EmitX86BuiltinExpr()
10365 Ops[1] = Ops[0]; in EmitX86BuiltinExpr()
10366 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
10380 return Builder.CreateShuffleVector(Ops[1], Ops[0], in EmitX86BuiltinExpr()
10390 unsigned NumElts = Ops[0]->getType()->getVectorNumElements(); in EmitX86BuiltinExpr()
10391 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
10400 return Builder.CreateShuffleVector(Ops[1], Ops[0], in EmitX86BuiltinExpr()
10412 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
10413 llvm::Type *Ty = Ops[0]->getType(); in EmitX86BuiltinExpr()
10429 return Builder.CreateShuffleVector(Ops[0], Ops[1], in EmitX86BuiltinExpr()
10438 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
10439 unsigned NumElts = Ops[0]->getType()->getVectorNumElements(); in EmitX86BuiltinExpr()
10451 OutOps[l] = llvm::ConstantAggregateZero::get(Ops[0]->getType()); in EmitX86BuiltinExpr()
10453 OutOps[l] = Ops[1]; in EmitX86BuiltinExpr()
10455 OutOps[l] = Ops[0]; in EmitX86BuiltinExpr()
10476 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
10477 llvm::Type *ResultType = Ops[0]->getType(); in EmitX86BuiltinExpr()
10496 Value *Cast = Builder.CreateBitCast(Ops[0], VecTy, "cast"); in EmitX86BuiltinExpr()
10501 return Builder.CreateBitCast(SV, Ops[0]->getType(), "cast"); in EmitX86BuiltinExpr()
10506 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
10507 llvm::Type *ResultType = Ops[0]->getType(); in EmitX86BuiltinExpr()
10526 Value *Cast = Builder.CreateBitCast(Ops[0], VecTy, "cast"); in EmitX86BuiltinExpr()
10537 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
10538 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
10541 return llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
10543 Value *In = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
10553 return Builder.CreateBitCast(SV, Ops[0]->getType()); in EmitX86BuiltinExpr()
10559 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
10560 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
10563 return llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
10565 Value *In = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
10575 return Builder.CreateBitCast(SV, Ops[0]->getType()); in EmitX86BuiltinExpr()
10584 Value *Ptr = Ops[0]; in EmitX86BuiltinExpr()
10585 Value *Src = Ops[1]; in EmitX86BuiltinExpr()
10623 return EmitX86FunnelShift(*this, Ops[0], Ops[0], Ops[1], false); in EmitX86BuiltinExpr()
10636 return EmitX86FunnelShift(*this, Ops[0], Ops[0], Ops[1], true); in EmitX86BuiltinExpr()
10655 return EmitX86Select(*this, Ops[0], Ops[1], Ops[2]); in EmitX86BuiltinExpr()
10658 Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0); in EmitX86BuiltinExpr()
10659 Value *B = Builder.CreateExtractElement(Ops[2], (uint64_t)0); in EmitX86BuiltinExpr()
10660 A = EmitX86ScalarSelect(*this, Ops[0], A, B); in EmitX86BuiltinExpr()
10661 return Builder.CreateInsertElement(Ops[1], A, (uint64_t)0); in EmitX86BuiltinExpr()
10675 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7; in EmitX86BuiltinExpr()
10676 return EmitX86MaskedCompare(*this, CC, true, Ops); in EmitX86BuiltinExpr()
10690 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7; in EmitX86BuiltinExpr()
10691 return EmitX86MaskedCompare(*this, CC, false, Ops); in EmitX86BuiltinExpr()
10698 Value *Or = EmitX86MaskLogic(*this, Instruction::Or, Ops); in EmitX86BuiltinExpr()
10699 Value *C = llvm::Constant::getAllOnesValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
10707 Value *Or = EmitX86MaskLogic(*this, Instruction::Or, Ops); in EmitX86BuiltinExpr()
10708 Value *C = llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
10750 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
10751 Value *LHS = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
10752 Value *RHS = getMaskVecValue(*this, Ops[1], NumElts); in EmitX86BuiltinExpr()
10778 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
10779 Value *LHS = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
10780 Value *RHS = getMaskVecValue(*this, Ops[1], NumElts); in EmitX86BuiltinExpr()
10783 return Builder.CreateBitCast(Res, Ops[0]->getType()); in EmitX86BuiltinExpr()
10789 return EmitX86MaskLogic(*this, Instruction::And, Ops); in EmitX86BuiltinExpr()
10794 return EmitX86MaskLogic(*this, Instruction::And, Ops, true); in EmitX86BuiltinExpr()
10799 return EmitX86MaskLogic(*this, Instruction::Or, Ops); in EmitX86BuiltinExpr()
10804 return EmitX86MaskLogic(*this, Instruction::Xor, Ops, true); in EmitX86BuiltinExpr()
10809 return EmitX86MaskLogic(*this, Instruction::Xor, Ops); in EmitX86BuiltinExpr()
10814 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
10815 Value *Res = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
10817 Ops[0]->getType()); in EmitX86BuiltinExpr()
10826 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
10827 Value *Res = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
10828 return Builder.CreateBitCast(Res, Ops[0]->getType()); in EmitX86BuiltinExpr()
10834 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
10835 Value *LHS = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
10836 Value *RHS = getMaskVecValue(*this, Ops[1], NumElts); in EmitX86BuiltinExpr()
10851 return Builder.CreateBitCast(Res, Ops[0]->getType()); in EmitX86BuiltinExpr()
10860 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType()); in EmitX86BuiltinExpr()
10861 return Builder.CreateCall(F, {Ops[0],Builder.getInt1(false)}); in EmitX86BuiltinExpr()
10865 Value *A = Builder.CreateExtractElement(Ops[0], (uint64_t)0); in EmitX86BuiltinExpr()
10868 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0); in EmitX86BuiltinExpr()
10872 unsigned CC = cast<llvm::ConstantInt>(Ops[4])->getZExtValue(); in EmitX86BuiltinExpr()
10879 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
10881 Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0); in EmitX86BuiltinExpr()
10884 Value *Src = Builder.CreateExtractElement(Ops[2], (uint64_t)0); in EmitX86BuiltinExpr()
10885 A = EmitX86ScalarSelect(*this, Ops[3], A, Src); in EmitX86BuiltinExpr()
10886 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0); in EmitX86BuiltinExpr()
10894 if (Ops.size() == 2) { in EmitX86BuiltinExpr()
10895 unsigned CC = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
10902 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
10905 Function *F = CGM.getIntrinsic(Intrinsic::sqrt, Ops[0]->getType()); in EmitX86BuiltinExpr()
10906 return Builder.CreateCall(F, Ops[0]); in EmitX86BuiltinExpr()
10920 return EmitX86Abs(*this, Ops); in EmitX86BuiltinExpr()
10934 return EmitX86MinMax(*this, ICmpInst::ICMP_SGT, Ops); in EmitX86BuiltinExpr()
10947 return EmitX86MinMax(*this, ICmpInst::ICMP_UGT, Ops); in EmitX86BuiltinExpr()
10960 return EmitX86MinMax(*this, ICmpInst::ICMP_SLT, Ops); in EmitX86BuiltinExpr()
10973 return EmitX86MinMax(*this, ICmpInst::ICMP_ULT, Ops); in EmitX86BuiltinExpr()
10978 return EmitX86Muldq(*this, /*IsSigned*/false, Ops); in EmitX86BuiltinExpr()
10983 return EmitX86Muldq(*this, /*IsSigned*/true, Ops); in EmitX86BuiltinExpr()
10991 return EmitX86Ternlog(*this, /*ZeroMask*/false, Ops); in EmitX86BuiltinExpr()
10999 return EmitX86Ternlog(*this, /*ZeroMask*/true, Ops); in EmitX86BuiltinExpr()
11010 return EmitX86FunnelShift(*this, Ops[0], Ops[1], Ops[2], false); in EmitX86BuiltinExpr()
11022 return EmitX86FunnelShift(*this, Ops[1], Ops[0], Ops[2], true); in EmitX86BuiltinExpr()
11033 return EmitX86FunnelShift(*this, Ops[0], Ops[1], Ops[2], false); in EmitX86BuiltinExpr()
11045 return EmitX86FunnelShift(*this, Ops[1], Ops[0], Ops[2], true); in EmitX86BuiltinExpr()
11051 Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast"); in EmitX86BuiltinExpr()
11053 return Builder.CreateCall(F, Ops, "pswapd"); in EmitX86BuiltinExpr()
11086 Ops[0]); in EmitX86BuiltinExpr()
11111 { Ops[0], Ops[1], Ops[2] }); in EmitX86BuiltinExpr()
11113 Ops[3]); in EmitX86BuiltinExpr()
11123 unsigned NumElts = Ops[0]->getType()->getVectorNumElements(); in EmitX86BuiltinExpr()
11124 Value *MaskIn = Ops[2]; in EmitX86BuiltinExpr()
11125 Ops.erase(&Ops[2]); in EmitX86BuiltinExpr()
11150 Value *Fpclass = Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
11171 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
11177 unsigned NumElts = Ops[0]->getType()->getVectorNumElements(); in EmitX86BuiltinExpr()
11178 Value *MaskIn = Ops[2]; in EmitX86BuiltinExpr()
11179 Ops.erase(&Ops[2]); in EmitX86BuiltinExpr()
11195 Value *Shufbit = Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
11242 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x1f; in EmitX86BuiltinExpr()
11293 unsigned NumElts = Ops[0]->getType()->getVectorNumElements(); in EmitX86BuiltinExpr()
11294 Value *Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
11295 return EmitX86MaskedCompareResult(*this, Cmp, NumElts, Ops[3]); in EmitX86BuiltinExpr()
11340 Value *LHS = Builder.CreateIntCast(Ops[0], Int64Ty, isSigned); in EmitX86BuiltinExpr()
11341 Value *RHS = Builder.CreateIntCast(Ops[1], Int64Ty, isSigned); in EmitX86BuiltinExpr()
11352 Value *LHS = Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned); in EmitX86BuiltinExpr()
11353 Value *RHS = Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned); in EmitX86BuiltinExpr()
11387 Builder.CreateShl(Builder.CreateZExt(Ops[1], Int128Ty), 64), in EmitX86BuiltinExpr()
11388 Builder.CreateZExt(Ops[0], Int128Ty)); in EmitX86BuiltinExpr()
11389 Value *Amt = Builder.CreateAnd(Builder.CreateZExt(Ops[2], Int128Ty), in EmitX86BuiltinExpr()
11438 Builder.CreateBitCast(Ops[0], Int128PtrTy); in EmitX86BuiltinExpr()
11439 Value *ExchangeHigh128 = Builder.CreateZExt(Ops[1], Int128Ty); in EmitX86BuiltinExpr()
11440 Value *ExchangeLow128 = Builder.CreateZExt(Ops[2], Int128Ty); in EmitX86BuiltinExpr()
11441 Address ComparandResult(Builder.CreateBitCast(Ops[3], Int128PtrTy), in EmitX86BuiltinExpr()
11471 return Builder.CreateMemSet(Ops[0], Ops[1], Ops[2], 1, true); in EmitX86BuiltinExpr()
11494 Builder.CreateIntToPtr(Ops[0], llvm::PointerType::get(IntTy, 257)); in EmitX86BuiltinExpr()
11506 Builder.CreateIntToPtr(Ops[0], llvm::PointerType::get(IntTy, 256)); in EmitX86BuiltinExpr()
11518 return EmitX86AddSubSatExpr(*this, Ops, true, true); in EmitX86BuiltinExpr()
11525 return EmitX86AddSubSatExpr(*this, Ops, false, true); in EmitX86BuiltinExpr()
11532 return EmitX86AddSubSatExpr(*this, Ops, true, false); in EmitX86BuiltinExpr()
11539 return EmitX86AddSubSatExpr(*this, Ops, false, false); in EmitX86BuiltinExpr()
11545 SmallVector<Value*, 4> Ops; in EmitPPCBuiltinExpr() local
11548 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitPPCBuiltinExpr()
11577 Ops[0] = Builder.CreateBitCast(Ops[0], Int8PtrTy); in EmitPPCBuiltinExpr()
11579 Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); in EmitPPCBuiltinExpr()
11580 Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]); in EmitPPCBuiltinExpr()
11581 Ops.pop_back(); in EmitPPCBuiltinExpr()
11627 return Builder.CreateCall(F, Ops, ""); in EmitPPCBuiltinExpr()
11645 Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); in EmitPPCBuiltinExpr()
11647 Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy); in EmitPPCBuiltinExpr()
11648 Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]); in EmitPPCBuiltinExpr()
11649 Ops.pop_back(); in EmitPPCBuiltinExpr()
11689 return Builder.CreateCall(F, Ops, ""); in EmitPPCBuiltinExpr()
11823 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]); in EmitPPCBuiltinExpr()
11834 std::swap(Ops[0], Ops[1]); in EmitPPCBuiltinExpr()
11838 Ops[1] = Builder.CreateBitCast(Ops[1], llvm::VectorType::get(Int64Ty, 2)); in EmitPPCBuiltinExpr()
11848 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2)); in EmitPPCBuiltinExpr()
11849 Ops[0] = Builder.CreateShuffleVector(Ops[0], Ops[0], ShuffleMask); in EmitPPCBuiltinExpr()
11856 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4)); in EmitPPCBuiltinExpr()
11857 Ops[2] = ConstantInt::getSigned(Int32Ty, Index); in EmitPPCBuiltinExpr()
11858 return Builder.CreateCall(F, Ops); in EmitPPCBuiltinExpr()
11865 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2)); in EmitPPCBuiltinExpr()
11869 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[1]); in EmitPPCBuiltinExpr()
11878 Ops[1] = ConstantInt::getSigned(Int32Ty, Index); in EmitPPCBuiltinExpr()
11881 Value *Call = Builder.CreateCall(F, Ops); in EmitPPCBuiltinExpr()
11892 Ops[1] = ConstantInt::getSigned(Int32Ty, Index); in EmitPPCBuiltinExpr()
11893 return Builder.CreateCall(F, Ops); in EmitPPCBuiltinExpr()
11898 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]); in EmitPPCBuiltinExpr()
11902 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2)); in EmitPPCBuiltinExpr()
11903 Ops[1] = Builder.CreateBitCast(Ops[1], llvm::VectorType::get(Int64Ty, 2)); in EmitPPCBuiltinExpr()
11916 Builder.CreateShuffleVector(Ops[0], Ops[1], ShuffleMask); in EmitPPCBuiltinExpr()
11923 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]); in EmitPPCBuiltinExpr()
11926 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4)); in EmitPPCBuiltinExpr()
11927 Ops[1] = Builder.CreateBitCast(Ops[1], llvm::VectorType::get(Int32Ty, 4)); in EmitPPCBuiltinExpr()
11957 Builder.CreateShuffleVector(Ops[0], Ops[1], ShuffleMask); in EmitPPCBuiltinExpr()
11966 llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), 2)); in EmitPPCBuiltinExpr()
11968 UndefValue, Ops[0], (uint64_t)(isLittleEndian ? 1 : 0)); in EmitPPCBuiltinExpr()
11969 Res = Builder.CreateInsertElement(Res, Ops[1], in EmitPPCBuiltinExpr()
11975 ConstantInt *Index = cast<ConstantInt>(Ops[1]); in EmitPPCBuiltinExpr()
11977 Ops[0], llvm::VectorType::get(ConvertType(E->getType()), 2)); in EmitPPCBuiltinExpr()
13312 SmallVector<llvm::Value *, 4> Ops; in EmitHexagonBuiltinExpr() local
13323 Ops = { Base, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2)), in EmitHexagonBuiltinExpr()
13326 Ops = { Base, EmitScalarExpr(E->getArg(1)), in EmitHexagonBuiltinExpr()
13329 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(IntID), Ops); in EmitHexagonBuiltinExpr()
13348 Ops = { Base, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2)), in EmitHexagonBuiltinExpr()
13351 Ops = { Base, EmitScalarExpr(E->getArg(1)), in EmitHexagonBuiltinExpr()
13354 llvm::Value *NewBase = Builder.CreateCall(CGM.getIntrinsic(IntID), Ops); in EmitHexagonBuiltinExpr()
13384 Ops = {BaseAddress, EmitScalarExpr(E->getArg(2))}; in EmitHexagonBuiltinExpr()
13386 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(IntID), Ops); in EmitHexagonBuiltinExpr()
13417 Ops = { EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), QLd }; in EmitHexagonBuiltinExpr()
13418 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitHexagonBuiltinExpr()
13439 Ops = { EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), QLd }; in EmitHexagonBuiltinExpr()
13440 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitHexagonBuiltinExpr()