Lines Matching refs:Ops
6133 Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops, in EmitNeonCall() argument
6143 Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift); in EmitNeonCall()
6145 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name); in EmitNeonCall()
6149 return Builder.CreateConstrainedFPCall(F, Ops, name); in EmitNeonCall()
6151 return Builder.CreateCall(F, Ops, name); in EmitNeonCall()
7340 SmallVectorImpl<Value *> &Ops, const CallExpr *E) { in EmitCommonNeonSISDBuiltinExpr() argument
7362 std::swap(Ops[0], Ops[1]); in EmitCommonNeonSISDBuiltinExpr()
7378 if (Ops[j]->getType()->getPrimitiveSizeInBits() == in EmitCommonNeonSISDBuiltinExpr()
7382 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy()); in EmitCommonNeonSISDBuiltinExpr()
7385 Ops[j] = CGF.Builder.CreateTruncOrBitCast( in EmitCommonNeonSISDBuiltinExpr()
7386 Ops[j], cast<llvm::VectorType>(ArgTy)->getElementType()); in EmitCommonNeonSISDBuiltinExpr()
7387 Ops[j] = in EmitCommonNeonSISDBuiltinExpr()
7388 CGF.Builder.CreateInsertElement(PoisonValue::get(ArgTy), Ops[j], C0); in EmitCommonNeonSISDBuiltinExpr()
7391 Value *Result = CGF.EmitNeonCall(F, Ops, s); in EmitCommonNeonSISDBuiltinExpr()
7403 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1, in EmitCommonNeonBuiltinExpr() argument
7446 Ops[0] = Builder.CreateBitCast(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
7447 return EmitNeonSplat(Ops[0], cast<ConstantInt>(Ops[1]), NumElements); in EmitCommonNeonBuiltinExpr()
7459 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs"); in EmitCommonNeonBuiltinExpr()
7460 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs"); in EmitCommonNeonBuiltinExpr()
7464 Ops[0] = Builder.CreateBitCast(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
7465 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); in EmitCommonNeonBuiltinExpr()
7466 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
7467 return Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
7474 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
7475 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); in EmitCommonNeonBuiltinExpr()
7476 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn"); in EmitCommonNeonBuiltinExpr()
7481 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn"); in EmitCommonNeonBuiltinExpr()
7484 return Builder.CreateTrunc(Ops[0], VTy, "vaddhn"); in EmitCommonNeonBuiltinExpr()
7490 std::swap(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
7512 return EmitNeonCall(F, Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7516 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ, in EmitCommonNeonBuiltinExpr()
7520 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE, in EmitCommonNeonBuiltinExpr()
7524 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE, in EmitCommonNeonBuiltinExpr()
7528 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT, in EmitCommonNeonBuiltinExpr()
7532 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT, in EmitCommonNeonBuiltinExpr()
7538 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef())); in EmitCommonNeonBuiltinExpr()
7542 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
7545 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") in EmitCommonNeonBuiltinExpr()
7546 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); in EmitCommonNeonBuiltinExpr()
7551 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
7554 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") in EmitCommonNeonBuiltinExpr()
7555 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); in EmitCommonNeonBuiltinExpr()
7562 return EmitNeonCall(F, Ops, "vcvt_n"); in EmitCommonNeonBuiltinExpr()
7571 return EmitNeonCall(F, Ops, "vcvt_n"); in EmitCommonNeonBuiltinExpr()
7587 return EmitNeonCall(F, Ops, "vcvt_n"); in EmitCommonNeonBuiltinExpr()
7601 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); in EmitCommonNeonBuiltinExpr()
7602 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt") in EmitCommonNeonBuiltinExpr()
7603 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt"); in EmitCommonNeonBuiltinExpr()
7654 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7658 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7663 int CV = cast<ConstantInt>(Ops[2])->getSExtValue(); in EmitCommonNeonBuiltinExpr()
7668 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
7669 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
7670 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices, "vext"); in EmitCommonNeonBuiltinExpr()
7674 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
7675 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
7676 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
7681 {Ops[1], Ops[2], Ops[0]}); in EmitCommonNeonBuiltinExpr()
7686 Ops.push_back(getAlignmentValue32(PtrOp0)); in EmitCommonNeonBuiltinExpr()
7687 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1"); in EmitCommonNeonBuiltinExpr()
7697 Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN"); in EmitCommonNeonBuiltinExpr()
7698 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitCommonNeonBuiltinExpr()
7715 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint); in EmitCommonNeonBuiltinExpr()
7716 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitCommonNeonBuiltinExpr()
7724 Ops[0] = Builder.CreateInsertElement(V, Ld, CI); in EmitCommonNeonBuiltinExpr()
7725 return EmitNeonSplat(Ops[0], CI); in EmitCommonNeonBuiltinExpr()
7735 for (unsigned I = 2; I < Ops.size() - 1; ++I) in EmitCommonNeonBuiltinExpr()
7736 Ops[I] = Builder.CreateBitCast(Ops[I], Ty); in EmitCommonNeonBuiltinExpr()
7737 Ops.push_back(getAlignmentValue32(PtrOp1)); in EmitCommonNeonBuiltinExpr()
7738 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), NameHint); in EmitCommonNeonBuiltinExpr()
7739 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitCommonNeonBuiltinExpr()
7744 Ops[0] = Builder.CreateBitCast(Ops[0], DTy); in EmitCommonNeonBuiltinExpr()
7746 return Builder.CreateZExt(Ops[0], Ty, "vmovl"); in EmitCommonNeonBuiltinExpr()
7747 return Builder.CreateSExt(Ops[0], Ty, "vmovl"); in EmitCommonNeonBuiltinExpr()
7752 Ops[0] = Builder.CreateBitCast(Ops[0], QTy); in EmitCommonNeonBuiltinExpr()
7753 return Builder.CreateTrunc(Ops[0], Ty, "vmovn"); in EmitCommonNeonBuiltinExpr()
7763 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); in EmitCommonNeonBuiltinExpr()
7773 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7783 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl"); in EmitCommonNeonBuiltinExpr()
7787 SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end()); in EmitCommonNeonBuiltinExpr()
7788 Ops[1] = in EmitCommonNeonBuiltinExpr()
7790 Ops.resize(2); in EmitCommonNeonBuiltinExpr()
7791 return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7805 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7814 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7818 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n", in EmitCommonNeonBuiltinExpr()
7822 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n", in EmitCommonNeonBuiltinExpr()
7829 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7835 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7838 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", in EmitCommonNeonBuiltinExpr()
7845 return EmitNeonCall(F, Ops, ""); in EmitCommonNeonBuiltinExpr()
7849 Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false); in EmitCommonNeonBuiltinExpr()
7850 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1], in EmitCommonNeonBuiltinExpr()
7855 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
7857 Ops[0] = Builder.CreateZExt(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
7859 Ops[0] = Builder.CreateSExt(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
7860 Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false); in EmitCommonNeonBuiltinExpr()
7861 return Builder.CreateShl(Ops[0], Ops[1], "vshll_n"); in EmitCommonNeonBuiltinExpr()
7866 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
7867 Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false); in EmitCommonNeonBuiltinExpr()
7869 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
7871 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
7872 return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n"); in EmitCommonNeonBuiltinExpr()
7876 return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n"); in EmitCommonNeonBuiltinExpr()
7892 Ops.push_back(getAlignmentValue32(PtrOp0)); in EmitCommonNeonBuiltinExpr()
7893 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, ""); in EmitCommonNeonBuiltinExpr()
7901 return EmitNeonCall(F, Ops, ""); in EmitCommonNeonBuiltinExpr()
7908 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); in EmitCommonNeonBuiltinExpr()
7909 return EmitNeonCall(F, Ops, ""); in EmitCommonNeonBuiltinExpr()
7922 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitCommonNeonBuiltinExpr()
7923 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, ""); in EmitCommonNeonBuiltinExpr()
7926 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, ""); in EmitCommonNeonBuiltinExpr()
7933 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
7934 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); in EmitCommonNeonBuiltinExpr()
7935 Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn"); in EmitCommonNeonBuiltinExpr()
7940 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn"); in EmitCommonNeonBuiltinExpr()
7943 return Builder.CreateTrunc(Ops[0], VTy, "vsubhn"); in EmitCommonNeonBuiltinExpr()
7947 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
7948 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
7957 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
7958 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn"); in EmitCommonNeonBuiltinExpr()
7965 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
7966 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
7967 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
7968 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], in EmitCommonNeonBuiltinExpr()
7970 return Builder.CreateSExt(Ops[0], Ty, "vtst"); in EmitCommonNeonBuiltinExpr()
7974 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
7975 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
7983 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
7984 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp"); in EmitCommonNeonBuiltinExpr()
7991 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty); in EmitCommonNeonBuiltinExpr()
7992 return EmitNeonCall(F, Ops, ""); in EmitCommonNeonBuiltinExpr()
7996 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
7997 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
8006 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
8007 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip"); in EmitCommonNeonBuiltinExpr()
8019 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vdot"); in EmitCommonNeonBuiltinExpr()
8026 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlal_low"); in EmitCommonNeonBuiltinExpr()
8033 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlsl_low"); in EmitCommonNeonBuiltinExpr()
8040 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlal_high"); in EmitCommonNeonBuiltinExpr()
8047 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlsl_high"); in EmitCommonNeonBuiltinExpr()
8054 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vmmla"); in EmitCommonNeonBuiltinExpr()
8060 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vusmmla"); in EmitCommonNeonBuiltinExpr()
8067 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vusdot"); in EmitCommonNeonBuiltinExpr()
8074 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vbfdot"); in EmitCommonNeonBuiltinExpr()
8079 return EmitNeonCall(F, Ops, "vcvtfp2bf"); in EmitCommonNeonBuiltinExpr()
8089 Value *Result = EmitNeonCall(F, Ops, NameHint); in EmitCommonNeonBuiltinExpr()
8120 static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops, in packTBLDVectorList() argument
8130 auto *TblTy = cast<llvm::FixedVectorType>(Ops[0]->getType()); in packTBLDVectorList()
8136 int PairPos = 0, End = Ops.size() - 1; in packTBLDVectorList()
8138 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], in packTBLDVectorList()
8139 Ops[PairPos+1], Indices, in packTBLDVectorList()
8148 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], in packTBLDVectorList()
8242 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) }; in EmitSpecialRegisterBuiltin() local
8243 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitSpecialRegisterBuiltin()
8406 Value *Ops[2]; in EmitARMBuiltinExpr() local
8408 Ops[i] = EmitScalarExpr(E->getArg(i)); in EmitARMBuiltinExpr()
8412 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); in EmitARMBuiltinExpr()
8714 SmallVector<Value*, 4> Ops; in EmitARMBuiltinExpr() local
8745 Ops.push_back(PtrOp0.getPointer()); in EmitARMBuiltinExpr()
8772 Ops.push_back(PtrOp1.getPointer()); in EmitARMBuiltinExpr()
8777 Ops.push_back(EmitScalarOrConstFoldImmArg(ICEArguments, i, E)); in EmitARMBuiltinExpr()
8797 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane"); in EmitARMBuiltinExpr()
8817 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitARMBuiltinExpr()
8820 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops, in EmitARMBuiltinExpr()
8823 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops, in EmitARMBuiltinExpr()
8826 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops, in EmitARMBuiltinExpr()
8829 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops, in EmitARMBuiltinExpr()
8833 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vcvtbfp2bf), Ops, in EmitARMBuiltinExpr()
8844 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0], in EmitARMBuiltinExpr()
8845 Ops[3], Ops[4], Ops[5]}); in EmitARMBuiltinExpr()
8872 return Builder.CreateCall(F, Ops, "vcvtr"); in EmitARMBuiltinExpr()
8895 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch); in EmitARMBuiltinExpr()
8905 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
8906 int Lane = cast<ConstantInt>(Ops[2])->getZExtValue(); in EmitARMBuiltinExpr()
8908 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); in EmitARMBuiltinExpr()
8914 Value *Ld = Builder.CreateCall(F, {Ops[0], Align}); in EmitARMBuiltinExpr()
8917 return Builder.CreateShuffleVector(Ops[1], Ld, Indices, "vld1q_lane"); in EmitARMBuiltinExpr()
8921 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
8924 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane"); in EmitARMBuiltinExpr()
8929 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n", in EmitARMBuiltinExpr()
8933 Ops, "vqrshrun_n", 1, true); in EmitARMBuiltinExpr()
8936 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n", in EmitARMBuiltinExpr()
8940 Ops, "vqshrun_n", 1, true); in EmitARMBuiltinExpr()
8944 Ops, "vrecpe"); in EmitARMBuiltinExpr()
8947 Ops, "vrshrn_n", 1, true); in EmitARMBuiltinExpr()
8950 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitARMBuiltinExpr()
8951 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
8952 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true); in EmitARMBuiltinExpr()
8954 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]}); in EmitARMBuiltinExpr()
8955 return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n"); in EmitARMBuiltinExpr()
8962 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift); in EmitARMBuiltinExpr()
8964 Ops, "vsli_n"); in EmitARMBuiltinExpr()
8967 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitARMBuiltinExpr()
8968 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); in EmitARMBuiltinExpr()
8969 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitARMBuiltinExpr()
8974 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
8975 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2])); in EmitARMBuiltinExpr()
8976 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); in EmitARMBuiltinExpr()
8977 Ops[2] = getAlignmentValue32(PtrOp0); in EmitARMBuiltinExpr()
8978 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()}; in EmitARMBuiltinExpr()
8980 Tys), Ops); in EmitARMBuiltinExpr()
8984 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
8985 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); in EmitARMBuiltinExpr()
8986 return Builder.CreateStore(Ops[1], in EmitARMBuiltinExpr()
8987 PtrOp0.withElementType(Ops[1]->getType())); in EmitARMBuiltinExpr()
8991 Ops, "vtbl1"); in EmitARMBuiltinExpr()
8994 Ops, "vtbl2"); in EmitARMBuiltinExpr()
8997 Ops, "vtbl3"); in EmitARMBuiltinExpr()
9000 Ops, "vtbl4"); in EmitARMBuiltinExpr()
9003 Ops, "vtbx1"); in EmitARMBuiltinExpr()
9006 Ops, "vtbx2"); in EmitARMBuiltinExpr()
9009 Ops, "vtbx3"); in EmitARMBuiltinExpr()
9012 Ops, "vtbx4"); in EmitARMBuiltinExpr()
9160 llvm::SmallVector<Value *, 4> Ops; in EmitARMMVEBuiltinExpr() local
9179 Ops.push_back(EmitScalarExpr(Addr)); in EmitARMMVEBuiltinExpr()
9183 Value *LoadResult = Builder.CreateCall(F, Ops); in EmitARMMVEBuiltinExpr()
9197 llvm::SmallVector<Value *, 4> Ops; in EmitARMMVEBuiltinExpr() local
9201 Ops.push_back(EmitScalarExpr(Addr)); in EmitARMMVEBuiltinExpr()
9222 Ops.push_back(Builder.CreateExtractValue(Mvec, {0, i})); in EmitARMMVEBuiltinExpr()
9227 Ops.push_back(llvm::ConstantInt::get(Int32Ty, i)); in EmitARMMVEBuiltinExpr()
9228 ToReturn = Builder.CreateCall(F, Ops); in EmitARMMVEBuiltinExpr()
9229 Ops.pop_back(); in EmitARMMVEBuiltinExpr()
9250 SmallVectorImpl<Value *> &Ops, in EmitAArch64TblBuiltinExpr() argument
9307 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(0, 1), nullptr, Ops[1], in EmitAArch64TblBuiltinExpr()
9311 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(0, 2), nullptr, Ops[2], in EmitAArch64TblBuiltinExpr()
9315 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(0, 3), nullptr, Ops[3], in EmitAArch64TblBuiltinExpr()
9319 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(0, 4), nullptr, Ops[4], in EmitAArch64TblBuiltinExpr()
9324 packTBLDVectorList(CGF, ArrayRef(Ops).slice(1, 1), nullptr, Ops[2], Ty, in EmitAArch64TblBuiltinExpr()
9328 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV); in EmitAArch64TblBuiltinExpr()
9331 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); in EmitAArch64TblBuiltinExpr()
9336 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(1, 2), Ops[0], Ops[3], in EmitAArch64TblBuiltinExpr()
9341 packTBLDVectorList(CGF, ArrayRef(Ops).slice(1, 3), nullptr, Ops[4], Ty, in EmitAArch64TblBuiltinExpr()
9345 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4], in EmitAArch64TblBuiltinExpr()
9349 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); in EmitAArch64TblBuiltinExpr()
9354 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(1, 4), Ops[0], Ops[5], in EmitAArch64TblBuiltinExpr()
9388 return CGF.EmitNeonCall(F, Ops, s); in EmitAArch64TblBuiltinExpr()
9577 SmallVectorImpl<Value *> &Ops, in EmitSVEGatherLoad() argument
9584 if (Ops[1]->getType()->isVectorTy()) in EmitSVEGatherLoad()
9588 F = CGM.getIntrinsic(IntID, {OverloadedTy, Ops[1]->getType()}); in EmitSVEGatherLoad()
9603 Ops[0] = EmitSVEPredicateCast( in EmitSVEGatherLoad()
9604 Ops[0], cast<llvm::ScalableVectorType>(F->getArg(0)->getType())); in EmitSVEGatherLoad()
9609 if (Ops.size() == 2) { in EmitSVEGatherLoad()
9610 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset"); in EmitSVEGatherLoad()
9611 Ops.push_back(ConstantInt::get(Int64Ty, 0)); in EmitSVEGatherLoad()
9616 if (!TypeFlags.isByteIndexed() && Ops[1]->getType()->isVectorTy()) { in EmitSVEGatherLoad()
9619 Ops[2] = Builder.CreateShl(Ops[2], Log2_32(BytesPerElt)); in EmitSVEGatherLoad()
9622 Value *Call = Builder.CreateCall(F, Ops); in EmitSVEGatherLoad()
9631 SmallVectorImpl<Value *> &Ops, in EmitSVEScatterStore() argument
9639 Ops.insert(Ops.begin(), Ops.pop_back_val()); in EmitSVEScatterStore()
9642 if (Ops[2]->getType()->isVectorTy()) in EmitSVEScatterStore()
9646 F = CGM.getIntrinsic(IntID, {OverloadedTy, Ops[2]->getType()}); in EmitSVEScatterStore()
9657 if (Ops.size() == 3) { in EmitSVEScatterStore()
9658 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset"); in EmitSVEScatterStore()
9659 Ops.push_back(ConstantInt::get(Int64Ty, 0)); in EmitSVEScatterStore()
9664 Ops[0] = Builder.CreateTrunc(Ops[0], OverloadedTy); in EmitSVEScatterStore()
9673 Ops[1] = EmitSVEPredicateCast( in EmitSVEScatterStore()
9674 Ops[1], cast<llvm::ScalableVectorType>(F->getArg(1)->getType())); in EmitSVEScatterStore()
9678 if (!TypeFlags.isByteIndexed() && Ops[2]->getType()->isVectorTy()) { in EmitSVEScatterStore()
9681 Ops[3] = Builder.CreateShl(Ops[3], Log2_32(BytesPerElt)); in EmitSVEScatterStore()
9684 return Builder.CreateCall(F, Ops); in EmitSVEScatterStore()
9688 SmallVectorImpl<Value *> &Ops, in EmitSVEGatherPrefetch() argument
9692 auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Ops[1]->getType()); in EmitSVEGatherPrefetch()
9694 OverloadedTy = cast<llvm::ScalableVectorType>(Ops[2]->getType()); in EmitSVEGatherPrefetch()
9697 Ops[0] = EmitSVEPredicateCast(Ops[0], OverloadedTy); in EmitSVEGatherPrefetch()
9700 if (Ops[1]->getType()->isVectorTy()) { in EmitSVEGatherPrefetch()
9701 if (Ops.size() == 3) { in EmitSVEGatherPrefetch()
9703 Ops.push_back(ConstantInt::get(Int64Ty, 0)); in EmitSVEGatherPrefetch()
9706 std::swap(Ops[2], Ops[3]); in EmitSVEGatherPrefetch()
9712 Ops[2] = Builder.CreateShl(Ops[2], Log2_32(BytesPerElt)); in EmitSVEGatherPrefetch()
9717 return Builder.CreateCall(F, Ops); in EmitSVEGatherPrefetch()
9721 SmallVectorImpl<Value*> &Ops, in EmitSVEStructLoad() argument
9749 Value *Predicate = EmitSVEPredicateCast(Ops[0], VTy); in EmitSVEStructLoad()
9750 Value *BasePtr = Ops[1]; in EmitSVEStructLoad()
9753 if (Ops.size() > 2) in EmitSVEStructLoad()
9754 BasePtr = Builder.CreateGEP(VTy, BasePtr, Ops[2]); in EmitSVEStructLoad()
9769 SmallVectorImpl<Value*> &Ops, in EmitSVEStructStore() argument
9795 Value *Predicate = EmitSVEPredicateCast(Ops[0], VTy); in EmitSVEStructStore()
9796 Value *BasePtr = Ops[1]; in EmitSVEStructStore()
9799 if (Ops.size() > (2 + N)) in EmitSVEStructStore()
9800 BasePtr = Builder.CreateGEP(VTy, BasePtr, Ops[2]); in EmitSVEStructStore()
9805 for (unsigned I = Ops.size() - N; I < Ops.size(); ++I) in EmitSVEStructStore()
9806 Operands.push_back(Ops[I]); in EmitSVEStructStore()
9817 SmallVectorImpl<Value *> &Ops, in EmitSVEPMull() argument
9822 Ops[OpNo] = EmitSVEDupX(Ops[OpNo]); in EmitSVEPMull()
9826 Function *F = CGM.getIntrinsic(BuiltinID, Ops[0]->getType()); in EmitSVEPMull()
9827 Value *Call = Builder.CreateCall(F, {Ops[0], Ops[1]}); in EmitSVEPMull()
9835 ArrayRef<Value *> Ops, unsigned BuiltinID) { in EmitSVEMovl() argument
9838 return Builder.CreateCall(F, {Ops[0], Builder.getInt32(0)}); in EmitSVEMovl()
9842 SmallVectorImpl<Value *> &Ops, in EmitSVEPrefetchLoad() argument
9848 Value *Predicate = EmitSVEPredicateCast(Ops[0], MemoryTy); in EmitSVEPrefetchLoad()
9849 Value *BasePtr = Ops[1]; in EmitSVEPrefetchLoad()
9852 if (Ops.size() > 3) in EmitSVEPrefetchLoad()
9853 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]); in EmitSVEPrefetchLoad()
9855 Value *PrfOp = Ops.back(); in EmitSVEPrefetchLoad()
9863 SmallVectorImpl<Value *> &Ops, in EmitSVEMaskedLoad() argument
9890 Value *Predicate = EmitSVEPredicateCast(Ops[0], PredTy); in EmitSVEMaskedLoad()
9891 Value *BasePtr = Ops[1]; in EmitSVEMaskedLoad()
9894 if (Ops.size() > 2) in EmitSVEMaskedLoad()
9895 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]); in EmitSVEMaskedLoad()
9911 SmallVectorImpl<Value *> &Ops, in EmitSVEMaskedStore() argument
9919 auto VectorTy = cast<llvm::ScalableVectorType>(Ops.back()->getType()); in EmitSVEMaskedStore()
9937 Value *Predicate = EmitSVEPredicateCast(Ops[0], PredTy); in EmitSVEMaskedStore()
9938 Value *BasePtr = Ops[1]; in EmitSVEMaskedStore()
9941 if (Ops.size() == 4) in EmitSVEMaskedStore()
9942 BasePtr = Builder.CreateGEP(AddrMemoryTy, BasePtr, Ops[2]); in EmitSVEMaskedStore()
9946 IsQuadStore ? Ops.back() : Builder.CreateTrunc(Ops.back(), MemoryTy); in EmitSVEMaskedStore()
9958 SmallVectorImpl<Value *> &Ops, in EmitSMELd1St1() argument
9960 Ops[2] = EmitSVEPredicateCast( in EmitSMELd1St1()
9961 Ops[2], getSVEVectorForElementType(SVEBuiltinMemEltTy(TypeFlags))); in EmitSMELd1St1()
9964 NewOps.push_back(Ops[2]); in EmitSMELd1St1()
9966 llvm::Value *BasePtr = Ops[3]; in EmitSMELd1St1()
9970 if (Ops.size() == 5) { in EmitSMELd1St1()
9976 Builder.CreateMul(StreamingVectorLengthCall, Ops[4], "mulvl"); in EmitSMELd1St1()
9978 BasePtr = Builder.CreateGEP(Int8Ty, Ops[3], Mulvl); in EmitSMELd1St1()
9981 NewOps.push_back(Ops[0]); in EmitSMELd1St1()
9982 NewOps.push_back(Ops[1]); in EmitSMELd1St1()
9988 SmallVectorImpl<Value *> &Ops, in EmitSMEReadWrite() argument
9993 Ops[1] = EmitSVEPredicateCast(Ops[1], VecTy); in EmitSMEReadWrite()
9995 Ops[2] = EmitSVEPredicateCast(Ops[2], VecTy); in EmitSMEReadWrite()
9996 return Builder.CreateCall(F, Ops); in EmitSMEReadWrite()
10000 SmallVectorImpl<Value *> &Ops, in EmitSMEZero() argument
10003 if (Ops.size() == 0) in EmitSMEZero()
10004 Ops.push_back(llvm::ConstantInt::get(Int32Ty, 255)); in EmitSMEZero()
10006 return Builder.CreateCall(F, Ops); in EmitSMEZero()
10010 SmallVectorImpl<Value *> &Ops, in EmitSMELdrStr() argument
10012 if (Ops.size() == 2) in EmitSMELdrStr()
10013 Ops.push_back(Builder.getInt32(0)); in EmitSMELdrStr()
10015 Ops[2] = Builder.CreateIntCast(Ops[2], Int32Ty, true); in EmitSMELdrStr()
10017 return Builder.CreateCall(F, Ops); in EmitSMELdrStr()
10042 SmallVectorImpl<Value *> &Ops) { in InsertExplicitZeroOperand() argument
10044 Ops.insert(Ops.begin(), SplatZero); in InsertExplicitZeroOperand()
10048 SmallVectorImpl<Value *> &Ops) { in InsertExplicitUndefOperand() argument
10050 Ops.insert(Ops.begin(), SplatUndef); in InsertExplicitUndefOperand()
10056 ArrayRef<Value *> Ops) { in getSVEOverloadTypes() argument
10063 return {DefaultType, Ops[1]->getType()}; in getSVEOverloadTypes()
10066 return {getSVEPredType(TypeFlags), Ops[0]->getType()}; in getSVEOverloadTypes()
10069 return {Ops[0]->getType(), Ops.back()->getType()}; in getSVEOverloadTypes()
10073 return {ResultType, Ops[1]->getType()}; in getSVEOverloadTypes()
10081 ArrayRef<Value *> Ops) { in EmitSVETupleSetOrGet() argument
10085 unsigned I = cast<ConstantInt>(Ops[1])->getSExtValue(); in EmitSVETupleSetOrGet()
10087 TypeFlags.isTupleSet() ? Ops[2]->getType() : Ty); in EmitSVETupleSetOrGet()
10092 return Builder.CreateInsertVector(Ty, Ops[0], Ops[2], Idx); in EmitSVETupleSetOrGet()
10093 return Builder.CreateExtractVector(Ty, Ops[0], Idx); in EmitSVETupleSetOrGet()
10098 ArrayRef<Value *> Ops) { in EmitSVETupleCreate() argument
10101 auto *SrcTy = dyn_cast<llvm::ScalableVectorType>(Ops[0]->getType()); in EmitSVETupleCreate()
10104 for (unsigned I = 0; I < Ops.size(); I++) { in EmitSVETupleCreate()
10106 Call = Builder.CreateInsertVector(Ty, Call, Ops[I], Idx); in EmitSVETupleCreate()
10148 unsigned BuiltinID, const CallExpr *E, SmallVectorImpl<Value *> &Ops, in GetAArch64SVEProcessedOperands() argument
10175 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), *Result)); in GetAArch64SVEProcessedOperands()
10180 Ops.push_back(Arg); in GetAArch64SVEProcessedOperands()
10190 Ops.push_back(Arg); in GetAArch64SVEProcessedOperands()
10198 Ops.push_back(Builder.CreateExtractVector(NewVTy, Arg, Idx)); in GetAArch64SVEProcessedOperands()
10215 llvm::SmallVector<Value *, 4> Ops; in EmitAArch64SVEBuiltinExpr() local
10217 GetAArch64SVEProcessedOperands(BuiltinID, E, Ops, TypeFlags); in EmitAArch64SVEBuiltinExpr()
10220 return EmitSVEMaskedLoad(E, Ty, Ops, Builtin->LLVMIntrinsic, in EmitAArch64SVEBuiltinExpr()
10223 return EmitSVEMaskedStore(E, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
10225 return EmitSVEGatherLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
10227 return EmitSVEScatterStore(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
10229 return EmitSVEPrefetchLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
10231 return EmitSVEGatherPrefetch(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
10233 return EmitSVEStructLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
10235 return EmitSVEStructStore(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
10237 return EmitSVETupleSetOrGet(TypeFlags, Ty, Ops); in EmitAArch64SVEBuiltinExpr()
10239 return EmitSVETupleCreate(TypeFlags, Ty, Ops); in EmitAArch64SVEBuiltinExpr()
10244 InsertExplicitZeroOperand(Builder, Ty, Ops); in EmitAArch64SVEBuiltinExpr()
10247 InsertExplicitUndefOperand(Builder, Ty, Ops); in EmitAArch64SVEBuiltinExpr()
10252 Ops.push_back(Builder.getInt32(/*SV_ALL*/ 31)); in EmitAArch64SVEBuiltinExpr()
10254 Ops.insert(&Ops[1], Builder.getInt32(/*SV_ALL*/ 31)); in EmitAArch64SVEBuiltinExpr()
10257 for (unsigned i = 0, e = Ops.size(); i != e; ++i) in EmitAArch64SVEBuiltinExpr()
10258 if (auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType())) in EmitAArch64SVEBuiltinExpr()
10260 Ops[i] = EmitSVEPredicateCast(Ops[i], getSVEType(TypeFlags)); in EmitAArch64SVEBuiltinExpr()
10265 Ops[OpNo] = EmitSVEDupX(Ops[OpNo]); in EmitAArch64SVEBuiltinExpr()
10269 std::swap(Ops[1], Ops[2]); in EmitAArch64SVEBuiltinExpr()
10271 std::swap(Ops[1], Ops[2]); in EmitAArch64SVEBuiltinExpr()
10274 std::swap(Ops[1], Ops[2]); in EmitAArch64SVEBuiltinExpr()
10277 std::swap(Ops[1], Ops[3]); in EmitAArch64SVEBuiltinExpr()
10281 llvm::Type *OpndTy = Ops[1]->getType(); in EmitAArch64SVEBuiltinExpr()
10283 Ops[1] = Builder.CreateSelect(Ops[0], Ops[1], SplatZero); in EmitAArch64SVEBuiltinExpr()
10287 getSVEOverloadTypes(TypeFlags, Ty, Ops)); in EmitAArch64SVEBuiltinExpr()
10288 Value *Call = Builder.CreateCall(F, Ops); in EmitAArch64SVEBuiltinExpr()
10307 return Builder.CreateCall(CastFromSVCountF, Ops[0]); in EmitAArch64SVEBuiltinExpr()
10314 return Builder.CreateCall(CastToSVCountF, Ops[0]); in EmitAArch64SVEBuiltinExpr()
10325 bool IsSVCount = isa<TargetExtType>(Ops[0]->getType()); in EmitAArch64SVEBuiltinExpr()
10326 assert(((!IsSVCount || cast<TargetExtType>(Ops[0]->getType())->getName() == in EmitAArch64SVEBuiltinExpr()
10339 IsSVCount ? Builder.CreateCall(CastFromSVCountF, Ops[0]) : Ops[0]; in EmitAArch64SVEBuiltinExpr()
10340 llvm::Value *Ops1 = EmitSVEPredicateCast(Ops[1], OverloadedTy); in EmitAArch64SVEBuiltinExpr()
10341 llvm::Value *PSel = Builder.CreateCall(F, {Ops0, Ops1, Ops[2]}); in EmitAArch64SVEBuiltinExpr()
10349 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[1]}); in EmitAArch64SVEBuiltinExpr()
10357 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[0]}); in EmitAArch64SVEBuiltinExpr()
10363 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllb); in EmitAArch64SVEBuiltinExpr()
10368 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllb); in EmitAArch64SVEBuiltinExpr()
10373 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllt); in EmitAArch64SVEBuiltinExpr()
10378 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllt); in EmitAArch64SVEBuiltinExpr()
10384 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullt_pair); in EmitAArch64SVEBuiltinExpr()
10390 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullb_pair); in EmitAArch64SVEBuiltinExpr()
10397 Builder.CreateICmpNE(Ops[0], Constant::getNullValue(Ops[0]->getType())); in EmitAArch64SVEBuiltinExpr()
10421 unsigned NumOpnds = Ops.size(); in EmitAArch64SVEBuiltinExpr()
10429 llvm::Type *EltTy = Ops[0]->getType(); in EmitAArch64SVEBuiltinExpr()
10435 VecOps.push_back(Builder.CreateZExt(Ops[I], EltTy)); in EmitAArch64SVEBuiltinExpr()
10508 return Builder.CreateCall(F, Ops); in EmitAArch64SVEBuiltinExpr()
10523 return Builder.CreateInsertVector(Ty, Ops[0], Ops[1], Builder.getInt64(0)); in EmitAArch64SVEBuiltinExpr()
10538 return Builder.CreateExtractVector(Ty, Ops[0], Builder.getInt64(0)); in EmitAArch64SVEBuiltinExpr()
10553 Value *Insert = Builder.CreateInsertVector(Ty, PoisonValue::get(Ty), Ops[0], in EmitAArch64SVEBuiltinExpr()
10565 SmallVectorImpl<Value *> &Ops) { in swapCommutativeSMEOperands() argument
10585 std::swap(Ops[I + 1], Ops[I + 1 + MultiVec]); in swapCommutativeSMEOperands()
10593 llvm::SmallVector<Value *, 4> Ops; in EmitAArch64SMEBuiltinExpr() local
10595 GetAArch64SVEProcessedOperands(BuiltinID, E, Ops, TypeFlags); in EmitAArch64SMEBuiltinExpr()
10598 return EmitSMELd1St1(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SMEBuiltinExpr()
10600 return EmitSMEReadWrite(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SMEBuiltinExpr()
10603 return EmitSMEZero(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SMEBuiltinExpr()
10608 return EmitSMELdrStr(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SMEBuiltinExpr()
10611 swapCommutativeSMEOperands(BuiltinID, Ops); in EmitAArch64SMEBuiltinExpr()
10618 for (unsigned i = 0, e = Ops.size(); i != e; ++i) in EmitAArch64SMEBuiltinExpr()
10619 if (auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType())) in EmitAArch64SMEBuiltinExpr()
10621 Ops[i] = EmitSVEPredicateCast(Ops[i], getSVEType(TypeFlags)); in EmitAArch64SMEBuiltinExpr()
10627 Value *Call = Builder.CreateCall(F, Ops); in EmitAArch64SMEBuiltinExpr()
10836 Value *Ops[2]; in EmitAArch64BuiltinExpr() local
10838 Ops[i] = EmitScalarExpr(E->getArg(i)); in EmitAArch64BuiltinExpr()
10842 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); in EmitAArch64BuiltinExpr()
10953 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)}; in EmitAArch64BuiltinExpr() local
10954 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitAArch64BuiltinExpr()
11161 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) }; in EmitAArch64BuiltinExpr() local
11162 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitAArch64BuiltinExpr()
11221 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "x18")}; in EmitAArch64BuiltinExpr() local
11222 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitAArch64BuiltinExpr()
11245 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "x18")}; in EmitAArch64BuiltinExpr() local
11246 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitAArch64BuiltinExpr()
11344 llvm::SmallVector<Value*, 4> Ops; in EmitAArch64BuiltinExpr() local
11366 Ops.push_back(PtrOp0.getPointer()); in EmitAArch64BuiltinExpr()
11370 Ops.push_back(EmitScalarOrConstFoldImmArg(ICEArguments, i, E)); in EmitAArch64BuiltinExpr()
11378 Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1))); in EmitAArch64BuiltinExpr()
11379 Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E); in EmitAArch64BuiltinExpr()
11398 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11399 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, HalfTy), Ops, "vabs"); in EmitAArch64BuiltinExpr()
11402 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11403 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
11404 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
11405 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11407 return Builder.CreateBitCast(Ops[0], Int128Ty); in EmitAArch64BuiltinExpr()
11416 Value *Ptr = Ops[0]; in EmitAArch64BuiltinExpr()
11425 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11426 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; in EmitAArch64BuiltinExpr()
11429 Ops[0] = Builder.CreateBitCast(Ops[0], InTy); in EmitAArch64BuiltinExpr()
11431 return Builder.CreateUIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
11432 return Builder.CreateSIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
11442 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11445 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64) in EmitAArch64BuiltinExpr()
11447 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32) in EmitAArch64BuiltinExpr()
11451 Ops[0] = Builder.CreateBitCast(Ops[0], InTy); in EmitAArch64BuiltinExpr()
11453 return Builder.CreateUIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
11454 return Builder.CreateSIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
11470 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11494 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvt"); in EmitAArch64BuiltinExpr()
11495 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11505 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11513 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]); break; in EmitAArch64BuiltinExpr()
11515 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]); break; in EmitAArch64BuiltinExpr()
11517 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "facg"); in EmitAArch64BuiltinExpr()
11518 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11526 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11534 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n"); in EmitAArch64BuiltinExpr()
11535 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11543 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11548 Ops[0] = Builder.CreateSExt(Ops[0], InTy, "sext"); in EmitAArch64BuiltinExpr()
11552 Ops[0] = Builder.CreateZExt(Ops[0], InTy); in EmitAArch64BuiltinExpr()
11555 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n"); in EmitAArch64BuiltinExpr()
11597 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11599 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
11605 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11607 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
11613 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11615 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
11621 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11623 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
11629 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11631 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
11635 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11636 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
11637 Ops[0] = in EmitAArch64BuiltinExpr()
11638 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty)); in EmitAArch64BuiltinExpr()
11639 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqzd"); in EmitAArch64BuiltinExpr()
11655 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11656 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
11657 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); in EmitAArch64BuiltinExpr()
11659 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11661 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11662 return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd"); in EmitAArch64BuiltinExpr()
11678 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11679 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy); in EmitAArch64BuiltinExpr()
11680 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy); in EmitAArch64BuiltinExpr()
11682 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11684 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11685 return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd"); in EmitAArch64BuiltinExpr()
11701 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11702 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
11703 Ops[1] = Builder.CreateBitCast(Ops[1], HalfTy); in EmitAArch64BuiltinExpr()
11705 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11707 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11708 return Builder.CreateSExt(Ops[0], Int16Ty, "vcmpd"); in EmitAArch64BuiltinExpr()
11734 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11735 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
11736 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
11737 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11738 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd"); in EmitAArch64BuiltinExpr()
11742 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11743 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
11744 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
11745 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11746 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], in EmitAArch64BuiltinExpr()
11748 return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd"); in EmitAArch64BuiltinExpr()
11762 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
11763 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
11766 Ops[1] = in EmitAArch64BuiltinExpr()
11767 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(DoubleTy, 1)); in EmitAArch64BuiltinExpr()
11768 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
11769 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
11772 Ops[1] = in EmitAArch64BuiltinExpr()
11773 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(DoubleTy, 2)); in EmitAArch64BuiltinExpr()
11774 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
11775 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
11779 Ops[0] = in EmitAArch64BuiltinExpr()
11780 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int8Ty, 8)); in EmitAArch64BuiltinExpr()
11781 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
11785 Ops[0] = in EmitAArch64BuiltinExpr()
11786 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int8Ty, 16)); in EmitAArch64BuiltinExpr()
11787 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
11791 Ops[0] = in EmitAArch64BuiltinExpr()
11792 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int16Ty, 4)); in EmitAArch64BuiltinExpr()
11793 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
11797 Ops[0] = in EmitAArch64BuiltinExpr()
11798 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int16Ty, 8)); in EmitAArch64BuiltinExpr()
11799 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
11803 Ops[0] = in EmitAArch64BuiltinExpr()
11804 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int32Ty, 2)); in EmitAArch64BuiltinExpr()
11805 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
11808 Ops[0] = in EmitAArch64BuiltinExpr()
11809 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 2)); in EmitAArch64BuiltinExpr()
11810 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
11814 Ops[0] = in EmitAArch64BuiltinExpr()
11815 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int32Ty, 4)); in EmitAArch64BuiltinExpr()
11816 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
11820 Ops[0] = in EmitAArch64BuiltinExpr()
11821 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 1)); in EmitAArch64BuiltinExpr()
11822 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
11825 Ops[0] = in EmitAArch64BuiltinExpr()
11826 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 1)); in EmitAArch64BuiltinExpr()
11827 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
11831 Ops[0] = in EmitAArch64BuiltinExpr()
11832 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 2)); in EmitAArch64BuiltinExpr()
11833 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
11836 Ops[0] = in EmitAArch64BuiltinExpr()
11837 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 2)); in EmitAArch64BuiltinExpr()
11838 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
11841 Ops[0] = in EmitAArch64BuiltinExpr()
11842 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 1)); in EmitAArch64BuiltinExpr()
11843 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
11847 Ops[0] = in EmitAArch64BuiltinExpr()
11848 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 4)); in EmitAArch64BuiltinExpr()
11849 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
11853 Ops[0] = in EmitAArch64BuiltinExpr()
11854 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 2)); in EmitAArch64BuiltinExpr()
11855 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
11858 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11859 return Builder.CreateFAdd(Ops[0], Ops[1], "vaddh"); in EmitAArch64BuiltinExpr()
11861 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11862 return Builder.CreateFSub(Ops[0], Ops[1], "vsubh"); in EmitAArch64BuiltinExpr()
11864 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11865 return Builder.CreateFMul(Ops[0], Ops[1], "vmulh"); in EmitAArch64BuiltinExpr()
11867 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11868 return Builder.CreateFDiv(Ops[0], Ops[1], "vdivh"); in EmitAArch64BuiltinExpr()
11873 {EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2)), Ops[0]}); in EmitAArch64BuiltinExpr()
11880 {Neg, EmitScalarExpr(E->getArg(2)), Ops[0]}); in EmitAArch64BuiltinExpr()
11884 return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd"); in EmitAArch64BuiltinExpr()
11887 return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd"); in EmitAArch64BuiltinExpr()
11891 ProductOps.push_back(vectorWrapScalar16(Ops[1])); in EmitAArch64BuiltinExpr()
11894 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), in EmitAArch64BuiltinExpr()
11897 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); in EmitAArch64BuiltinExpr()
11902 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
11905 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11906 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
11908 Ops, "vqshlu_n"); in EmitAArch64BuiltinExpr()
11915 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11916 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
11917 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n"); in EmitAArch64BuiltinExpr()
11924 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11925 int SV = cast<ConstantInt>(Ops[1])->getSExtValue(); in EmitAArch64BuiltinExpr()
11926 Ops[1] = ConstantInt::get(Int64Ty, -SV); in EmitAArch64BuiltinExpr()
11927 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n"); in EmitAArch64BuiltinExpr()
11934 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
11935 Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2)))); in EmitAArch64BuiltinExpr()
11936 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty), in EmitAArch64BuiltinExpr()
11937 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)}); in EmitAArch64BuiltinExpr()
11938 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty)); in EmitAArch64BuiltinExpr()
11944 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n"); in EmitAArch64BuiltinExpr()
11949 Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), in EmitAArch64BuiltinExpr()
11959 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt), in EmitAArch64BuiltinExpr()
11964 Ops[1] = Builder.CreateAShr( in EmitAArch64BuiltinExpr()
11965 Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), in EmitAArch64BuiltinExpr()
11968 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11976 return Ops[0]; in EmitAArch64BuiltinExpr()
11977 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt), in EmitAArch64BuiltinExpr()
11979 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11985 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), in EmitAArch64BuiltinExpr()
11988 ProductOps.push_back(vectorWrapScalar16(Ops[1])); in EmitAArch64BuiltinExpr()
11989 ProductOps.push_back(vectorWrapScalar16(Ops[2])); in EmitAArch64BuiltinExpr()
11991 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), in EmitAArch64BuiltinExpr()
11994 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); in EmitAArch64BuiltinExpr()
11995 Ops.pop_back(); in EmitAArch64BuiltinExpr()
12001 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
12006 ProductOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
12008 Ops[1] = in EmitAArch64BuiltinExpr()
12015 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
12021 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), in EmitAArch64BuiltinExpr()
12024 ProductOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
12025 ProductOps.push_back(Ops[2]); in EmitAArch64BuiltinExpr()
12026 Ops[1] = in EmitAArch64BuiltinExpr()
12029 Ops.pop_back(); in EmitAArch64BuiltinExpr()
12035 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
12040 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
12046 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
12073 Builtin->NameHint, Builtin->TypeModifier, E, Ops, in EmitAArch64BuiltinExpr()
12076 if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops, Arch)) in EmitAArch64BuiltinExpr()
12085 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
12086 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
12087 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
12089 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl"); in EmitAArch64BuiltinExpr()
12090 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl"); in EmitAArch64BuiltinExpr()
12091 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl"); in EmitAArch64BuiltinExpr()
12092 return Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
12098 Value *Addend = Ops[0]; in EmitAArch64BuiltinExpr()
12099 Value *Multiplicand = Ops[1]; in EmitAArch64BuiltinExpr()
12100 Value *LaneSource = Ops[2]; in EmitAArch64BuiltinExpr()
12101 Ops[0] = Multiplicand; in EmitAArch64BuiltinExpr()
12102 Ops[1] = LaneSource; in EmitAArch64BuiltinExpr()
12103 Ops[2] = Addend; in EmitAArch64BuiltinExpr()
12110 llvm::Constant *cst = cast<Constant>(Ops[3]); in EmitAArch64BuiltinExpr()
12112 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy); in EmitAArch64BuiltinExpr()
12113 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane"); in EmitAArch64BuiltinExpr()
12115 Ops.pop_back(); in EmitAArch64BuiltinExpr()
12118 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla"); in EmitAArch64BuiltinExpr()
12124 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
12125 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); in EmitAArch64BuiltinExpr()
12128 Ops[2] = Builder.CreateBitCast(Ops[2], VTy); in EmitAArch64BuiltinExpr()
12129 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); in EmitAArch64BuiltinExpr()
12133 DoubleTy, {Ops[1], Ops[2], Ops[0]}); in EmitAArch64BuiltinExpr()
12136 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
12137 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
12141 Ops[2] = Builder.CreateBitCast(Ops[2], STy); in EmitAArch64BuiltinExpr()
12143 cast<ConstantInt>(Ops[3])); in EmitAArch64BuiltinExpr()
12144 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane"); in EmitAArch64BuiltinExpr()
12148 {Ops[2], Ops[1], Ops[0]}); in EmitAArch64BuiltinExpr()
12151 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
12152 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
12154 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
12155 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3])); in EmitAArch64BuiltinExpr()
12158 {Ops[2], Ops[1], Ops[0]}); in EmitAArch64BuiltinExpr()
12166 Ops.push_back(EmitScalarExpr(E->getArg(3))); in EmitAArch64BuiltinExpr()
12168 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); in EmitAArch64BuiltinExpr()
12171 {Ops[1], Ops[2], Ops[0]}); in EmitAArch64BuiltinExpr()
12177 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); in EmitAArch64BuiltinExpr()
12183 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax"); in EmitAArch64BuiltinExpr()
12185 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12187 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmax"); in EmitAArch64BuiltinExpr()
12194 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin"); in EmitAArch64BuiltinExpr()
12196 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12198 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmin"); in EmitAArch64BuiltinExpr()
12205 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd"); in EmitAArch64BuiltinExpr()
12216 TmpOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
12219 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType()); in EmitAArch64BuiltinExpr()
12227 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin"); in EmitAArch64BuiltinExpr()
12233 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax"); in EmitAArch64BuiltinExpr()
12237 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm"); in EmitAArch64BuiltinExpr()
12239 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12241 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vminnm"); in EmitAArch64BuiltinExpr()
12245 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm"); in EmitAArch64BuiltinExpr()
12247 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12249 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmaxnm"); in EmitAArch64BuiltinExpr()
12251 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12253 Ops, "vrecps"); in EmitAArch64BuiltinExpr()
12256 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12258 Ops, "vrecps"); in EmitAArch64BuiltinExpr()
12260 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12262 Ops, "vrecps"); in EmitAArch64BuiltinExpr()
12265 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n"); in EmitAArch64BuiltinExpr()
12268 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n"); in EmitAArch64BuiltinExpr()
12271 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n"); in EmitAArch64BuiltinExpr()
12274 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n"); in EmitAArch64BuiltinExpr()
12277 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n"); in EmitAArch64BuiltinExpr()
12279 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12283 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrnda"); in EmitAArch64BuiltinExpr()
12290 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda"); in EmitAArch64BuiltinExpr()
12293 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12297 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndi"); in EmitAArch64BuiltinExpr()
12300 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12304 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndm"); in EmitAArch64BuiltinExpr()
12311 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm"); in EmitAArch64BuiltinExpr()
12314 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12318 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndn"); in EmitAArch64BuiltinExpr()
12325 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn"); in EmitAArch64BuiltinExpr()
12328 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12332 return EmitNeonCall(CGM.getIntrinsic(Int, FloatTy), Ops, "vrndn"); in EmitAArch64BuiltinExpr()
12335 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12339 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndp"); in EmitAArch64BuiltinExpr()
12346 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp"); in EmitAArch64BuiltinExpr()
12349 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12353 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndx"); in EmitAArch64BuiltinExpr()
12360 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx"); in EmitAArch64BuiltinExpr()
12363 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12367 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndz"); in EmitAArch64BuiltinExpr()
12373 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12375 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd32x"); in EmitAArch64BuiltinExpr()
12381 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12383 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd32z"); in EmitAArch64BuiltinExpr()
12389 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12391 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd64x"); in EmitAArch64BuiltinExpr()
12397 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12399 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd64z"); in EmitAArch64BuiltinExpr()
12406 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz"); in EmitAArch64BuiltinExpr()
12410 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
12412 return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") in EmitAArch64BuiltinExpr()
12413 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
12418 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); in EmitAArch64BuiltinExpr()
12420 return Builder.CreateFPExt(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
12426 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); in EmitAArch64BuiltinExpr()
12428 return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
12445 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtz"); in EmitAArch64BuiltinExpr()
12461 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta"); in EmitAArch64BuiltinExpr()
12477 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm"); in EmitAArch64BuiltinExpr()
12493 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn"); in EmitAArch64BuiltinExpr()
12509 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp"); in EmitAArch64BuiltinExpr()
12514 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx"); in EmitAArch64BuiltinExpr()
12520 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
12521 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract"); in EmitAArch64BuiltinExpr()
12522 Ops.pop_back(); in EmitAArch64BuiltinExpr()
12524 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmulx"); in EmitAArch64BuiltinExpr()
12532 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
12535 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); in EmitAArch64BuiltinExpr()
12536 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract"); in EmitAArch64BuiltinExpr()
12537 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
12547 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm"); in EmitAArch64BuiltinExpr()
12552 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm"); in EmitAArch64BuiltinExpr()
12555 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12559 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vsqrt"); in EmitAArch64BuiltinExpr()
12566 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
12567 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt"); in EmitAArch64BuiltinExpr()
12572 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit"); in EmitAArch64BuiltinExpr()
12583 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12584 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
12585 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
12595 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12596 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
12597 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
12607 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12608 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
12609 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
12619 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12620 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
12621 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
12628 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12629 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
12630 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
12637 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12638 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
12639 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
12646 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12647 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
12648 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
12655 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12656 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
12657 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
12664 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12665 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
12666 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
12673 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12674 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
12675 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
12682 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12683 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
12684 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
12691 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12692 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
12693 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
12700 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12701 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
12702 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
12709 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12710 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
12711 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
12718 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12719 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
12720 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
12727 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12728 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
12729 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
12736 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12737 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
12738 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
12745 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12746 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
12747 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
12754 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12755 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
12756 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
12763 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12764 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
12765 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
12772 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12773 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
12774 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
12781 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12782 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
12783 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
12790 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12791 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
12792 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
12799 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12800 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
12801 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
12808 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12809 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv"); in EmitAArch64BuiltinExpr()
12810 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
12817 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12818 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv"); in EmitAArch64BuiltinExpr()
12819 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
12826 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12827 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv"); in EmitAArch64BuiltinExpr()
12828 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
12835 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12836 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv"); in EmitAArch64BuiltinExpr()
12837 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
12840 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
12842 return Builder.CreateFMul(Ops[0], RHS); in EmitAArch64BuiltinExpr()
12849 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12850 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
12851 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
12858 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12859 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
12866 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12867 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
12868 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
12875 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12876 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
12883 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12884 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
12885 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
12892 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12893 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
12900 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12901 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
12902 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
12909 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12910 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
12916 return EmitNeonCall(Intrin, Ops, "vsri_n"); in EmitAArch64BuiltinExpr()
12922 return EmitNeonCall(Intrin, Ops, "vsli_n"); in EmitAArch64BuiltinExpr()
12926 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
12927 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); in EmitAArch64BuiltinExpr()
12928 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
12933 TmpOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
12934 TmpOps.push_back(Ops[2]); in EmitAArch64BuiltinExpr()
12937 Ops[0] = Builder.CreateBitCast(Ops[0], VTy); in EmitAArch64BuiltinExpr()
12938 return Builder.CreateAdd(Ops[0], tmp); in EmitAArch64BuiltinExpr()
12942 return Builder.CreateAlignedLoad(VTy, Ops[0], PtrOp0.getAlignment()); in EmitAArch64BuiltinExpr()
12946 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); in EmitAArch64BuiltinExpr()
12947 return Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.getAlignment()); in EmitAArch64BuiltinExpr()
12950 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
12951 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], in EmitAArch64BuiltinExpr()
12953 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane"); in EmitAArch64BuiltinExpr()
12957 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
12959 VTy->getElementType(), Ops[0], PtrOp0.getAlignment()); in EmitAArch64BuiltinExpr()
12961 Ops[0] = LI; in EmitAArch64BuiltinExpr()
12962 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vldap1_lane"); in EmitAArch64BuiltinExpr()
12967 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], in EmitAArch64BuiltinExpr()
12970 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI); in EmitAArch64BuiltinExpr()
12971 return EmitNeonSplat(Ops[0], CI); in EmitAArch64BuiltinExpr()
12975 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
12976 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); in EmitAArch64BuiltinExpr()
12977 return Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.getAlignment()); in EmitAArch64BuiltinExpr()
12980 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
12981 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); in EmitAArch64BuiltinExpr()
12983 Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.getAlignment()); in EmitAArch64BuiltinExpr()
12991 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); in EmitAArch64BuiltinExpr()
12992 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
12998 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); in EmitAArch64BuiltinExpr()
12999 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
13005 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); in EmitAArch64BuiltinExpr()
13006 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
13012 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); in EmitAArch64BuiltinExpr()
13013 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
13019 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); in EmitAArch64BuiltinExpr()
13020 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
13026 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); in EmitAArch64BuiltinExpr()
13027 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
13031 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
13033 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end()); in EmitAArch64BuiltinExpr()
13034 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
13035 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
13036 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); in EmitAArch64BuiltinExpr()
13037 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld2_lane"); in EmitAArch64BuiltinExpr()
13038 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
13042 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
13044 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end()); in EmitAArch64BuiltinExpr()
13045 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
13046 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
13047 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); in EmitAArch64BuiltinExpr()
13048 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); in EmitAArch64BuiltinExpr()
13049 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld3_lane"); in EmitAArch64BuiltinExpr()
13050 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
13054 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
13056 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end()); in EmitAArch64BuiltinExpr()
13057 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
13058 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
13059 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); in EmitAArch64BuiltinExpr()
13060 Ops[4] = Builder.CreateBitCast(Ops[4], Ty); in EmitAArch64BuiltinExpr()
13061 Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty); in EmitAArch64BuiltinExpr()
13062 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld4_lane"); in EmitAArch64BuiltinExpr()
13063 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
13067 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
13068 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() }; in EmitAArch64BuiltinExpr()
13070 Ops, ""); in EmitAArch64BuiltinExpr()
13074 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
13075 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty); in EmitAArch64BuiltinExpr()
13076 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; in EmitAArch64BuiltinExpr()
13078 Ops, ""); in EmitAArch64BuiltinExpr()
13082 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
13083 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; in EmitAArch64BuiltinExpr()
13085 Ops, ""); in EmitAArch64BuiltinExpr()
13089 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
13090 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); in EmitAArch64BuiltinExpr()
13091 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; in EmitAArch64BuiltinExpr()
13093 Ops, ""); in EmitAArch64BuiltinExpr()
13097 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
13098 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; in EmitAArch64BuiltinExpr()
13100 Ops, ""); in EmitAArch64BuiltinExpr()
13104 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
13105 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); in EmitAArch64BuiltinExpr()
13106 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() }; in EmitAArch64BuiltinExpr()
13108 Ops, ""); in EmitAArch64BuiltinExpr()
13112 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
13113 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
13122 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
13123 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn"); in EmitAArch64BuiltinExpr()
13130 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
13131 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
13139 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
13140 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp"); in EmitAArch64BuiltinExpr()
13147 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
13148 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
13157 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
13158 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip"); in EmitAArch64BuiltinExpr()
13165 Ops, "vtbl1"); in EmitAArch64BuiltinExpr()
13169 Ops, "vtbl2"); in EmitAArch64BuiltinExpr()
13173 Ops, "vtbl3"); in EmitAArch64BuiltinExpr()
13177 Ops, "vtbl4"); in EmitAArch64BuiltinExpr()
13181 Ops, "vtbx1"); in EmitAArch64BuiltinExpr()
13185 Ops, "vtbx2"); in EmitAArch64BuiltinExpr()
13189 Ops, "vtbx3"); in EmitAArch64BuiltinExpr()
13193 Ops, "vtbx4"); in EmitAArch64BuiltinExpr()
13198 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd"); in EmitAArch64BuiltinExpr()
13203 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd"); in EmitAArch64BuiltinExpr()
13317 BuildVector(ArrayRef<llvm::Value*> Ops) { in BuildVector() argument
13318 assert((Ops.size() & (Ops.size() - 1)) == 0 && in BuildVector()
13321 for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i) in BuildVector()
13322 AllConstants &= isa<Constant>(Ops[i]); in BuildVector()
13327 for (unsigned i = 0, e = Ops.size(); i != e; ++i) in BuildVector()
13328 CstOps.push_back(cast<Constant>(Ops[i])); in BuildVector()
13334 llvm::FixedVectorType::get(Ops[0]->getType(), Ops.size())); in BuildVector()
13336 for (unsigned i = 0, e = Ops.size(); i != e; ++i) in BuildVector()
13337 Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt64(i)); in BuildVector()
13363 static Value *EmitX86MaskedStore(CodeGenFunction &CGF, ArrayRef<Value *> Ops, in EmitX86MaskedStore() argument
13365 Value *Ptr = Ops[0]; in EmitX86MaskedStore()
13368 CGF, Ops[2], in EmitX86MaskedStore()
13369 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements()); in EmitX86MaskedStore()
13371 return CGF.Builder.CreateMaskedStore(Ops[1], Ptr, Alignment, MaskVec); in EmitX86MaskedStore()
13374 static Value *EmitX86MaskedLoad(CodeGenFunction &CGF, ArrayRef<Value *> Ops, in EmitX86MaskedLoad() argument
13376 llvm::Type *Ty = Ops[1]->getType(); in EmitX86MaskedLoad()
13377 Value *Ptr = Ops[0]; in EmitX86MaskedLoad()
13380 CGF, Ops[2], cast<llvm::FixedVectorType>(Ty)->getNumElements()); in EmitX86MaskedLoad()
13382 return CGF.Builder.CreateMaskedLoad(Ty, Ptr, Alignment, MaskVec, Ops[1]); in EmitX86MaskedLoad()
13386 ArrayRef<Value *> Ops) { in EmitX86ExpandLoad() argument
13387 auto *ResultTy = cast<llvm::VectorType>(Ops[1]->getType()); in EmitX86ExpandLoad()
13388 Value *Ptr = Ops[0]; in EmitX86ExpandLoad()
13391 CGF, Ops[2], cast<FixedVectorType>(ResultTy)->getNumElements()); in EmitX86ExpandLoad()
13395 return CGF.Builder.CreateCall(F, { Ptr, MaskVec, Ops[1] }); in EmitX86ExpandLoad()
13399 ArrayRef<Value *> Ops, in EmitX86CompressExpand() argument
13401 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType()); in EmitX86CompressExpand()
13403 Value *MaskVec = getMaskVecValue(CGF, Ops[2], ResultTy->getNumElements()); in EmitX86CompressExpand()
13408 return CGF.Builder.CreateCall(F, { Ops[0], Ops[1], MaskVec }); in EmitX86CompressExpand()
13412 ArrayRef<Value *> Ops) { in EmitX86CompressStore() argument
13413 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType()); in EmitX86CompressStore()
13414 Value *Ptr = Ops[0]; in EmitX86CompressStore()
13416 Value *MaskVec = getMaskVecValue(CGF, Ops[2], ResultTy->getNumElements()); in EmitX86CompressStore()
13420 return CGF.Builder.CreateCall(F, { Ops[1], Ptr, MaskVec }); in EmitX86CompressStore()
13424 ArrayRef<Value *> Ops, in EmitX86MaskLogic() argument
13426 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86MaskLogic()
13427 Value *LHS = getMaskVecValue(CGF, Ops[0], NumElts); in EmitX86MaskLogic()
13428 Value *RHS = getMaskVecValue(CGF, Ops[1], NumElts); in EmitX86MaskLogic()
13434 Ops[0]->getType()); in EmitX86MaskLogic()
13455 static Value *EmitX86vpcom(CodeGenFunction &CGF, ArrayRef<Value *> Ops, in EmitX86vpcom() argument
13457 Value *Op0 = Ops[0]; in EmitX86vpcom()
13458 Value *Op1 = Ops[1]; in EmitX86vpcom()
13460 uint64_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7; in EmitX86vpcom()
13547 bool Signed, ArrayRef<Value *> Ops) { in EmitX86MaskedCompare() argument
13548 assert((Ops.size() == 2 || Ops.size() == 4) && in EmitX86MaskedCompare()
13551 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86MaskedCompare()
13571 Cmp = CGF.Builder.CreateICmp(Pred, Ops[0], Ops[1]); in EmitX86MaskedCompare()
13575 if (Ops.size() == 4) in EmitX86MaskedCompare()
13576 MaskIn = Ops[3]; in EmitX86MaskedCompare()
13587 ArrayRef<Value *> Ops, bool IsSigned) { in EmitX86ConvertIntToFp() argument
13588 unsigned Rnd = cast<llvm::ConstantInt>(Ops[3])->getZExtValue(); in EmitX86ConvertIntToFp()
13589 llvm::Type *Ty = Ops[1]->getType(); in EmitX86ConvertIntToFp()
13595 Function *F = CGF.CGM.getIntrinsic(IID, { Ty, Ops[0]->getType() }); in EmitX86ConvertIntToFp()
13596 Res = CGF.Builder.CreateCall(F, { Ops[0], Ops[3] }); in EmitX86ConvertIntToFp()
13599 Res = IsSigned ? CGF.Builder.CreateSIToFP(Ops[0], Ty) in EmitX86ConvertIntToFp()
13600 : CGF.Builder.CreateUIToFP(Ops[0], Ty); in EmitX86ConvertIntToFp()
13603 return EmitX86Select(CGF, Ops[2], Res, Ops[1]); in EmitX86ConvertIntToFp()
13608 ArrayRef<Value *> Ops, unsigned BuiltinID, in EmitX86FMAExpr() argument
13663 Value *A = Ops[0]; in EmitX86FMAExpr()
13664 Value *B = Ops[1]; in EmitX86FMAExpr()
13665 Value *C = Ops[2]; in EmitX86FMAExpr()
13674 (cast<llvm::ConstantInt>(Ops.back())->getZExtValue() != (uint64_t)4 || in EmitX86FMAExpr()
13677 Res = CGF.Builder.CreateCall(Intr, {A, B, C, Ops.back() }); in EmitX86FMAExpr()
13700 MaskFalseVal = Ops[0]; in EmitX86FMAExpr()
13708 MaskFalseVal = Constant::getNullValue(Ops[0]->getType()); in EmitX86FMAExpr()
13722 MaskFalseVal = Ops[2]; in EmitX86FMAExpr()
13727 return EmitX86Select(CGF, Ops[3], Res, MaskFalseVal); in EmitX86FMAExpr()
13733 MutableArrayRef<Value *> Ops, Value *Upper, in EmitScalarFMAExpr() argument
13737 if (Ops.size() > 4) in EmitScalarFMAExpr()
13738 Rnd = cast<llvm::ConstantInt>(Ops[4])->getZExtValue(); in EmitScalarFMAExpr()
13741 Ops[2] = CGF.Builder.CreateFNeg(Ops[2]); in EmitScalarFMAExpr()
13743 Ops[0] = CGF.Builder.CreateExtractElement(Ops[0], (uint64_t)0); in EmitScalarFMAExpr()
13744 Ops[1] = CGF.Builder.CreateExtractElement(Ops[1], (uint64_t)0); in EmitScalarFMAExpr()
13745 Ops[2] = CGF.Builder.CreateExtractElement(Ops[2], (uint64_t)0); in EmitScalarFMAExpr()
13750 switch (Ops[0]->getType()->getPrimitiveSizeInBits()) { in EmitScalarFMAExpr()
13764 {Ops[0], Ops[1], Ops[2], Ops[4]}); in EmitScalarFMAExpr()
13768 Intrinsic::experimental_constrained_fma, Ops[0]->getType()); in EmitScalarFMAExpr()
13769 Res = CGF.Builder.CreateConstrainedFPCall(FMA, Ops.slice(0, 3)); in EmitScalarFMAExpr()
13771 Function *FMA = CGF.CGM.getIntrinsic(Intrinsic::fma, Ops[0]->getType()); in EmitScalarFMAExpr()
13772 Res = CGF.Builder.CreateCall(FMA, Ops.slice(0, 3)); in EmitScalarFMAExpr()
13775 if (Ops.size() > 3) { in EmitScalarFMAExpr()
13777 : Ops[PTIdx]; in EmitScalarFMAExpr()
13785 Res = EmitX86ScalarSelect(CGF, Ops[3], Res, PassThru); in EmitScalarFMAExpr()
13791 ArrayRef<Value *> Ops) { in EmitX86Muldq() argument
13792 llvm::Type *Ty = Ops[0]->getType(); in EmitX86Muldq()
13796 Value *LHS = CGF.Builder.CreateBitCast(Ops[0], Ty); in EmitX86Muldq()
13797 Value *RHS = CGF.Builder.CreateBitCast(Ops[1], Ty); in EmitX86Muldq()
13820 ArrayRef<Value *> Ops) { in EmitX86Ternlog() argument
13821 llvm::Type *Ty = Ops[0]->getType(); in EmitX86Ternlog()
13842 Ops.drop_back()); in EmitX86Ternlog()
13843 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) : Ops[0]; in EmitX86Ternlog()
13844 return EmitX86Select(CGF, Ops[4], Ternlog, PassThru); in EmitX86Ternlog()
13863 ArrayRef<Value *> Ops, in EmitX86CvtF16ToFloatExpr() argument
13865 assert((Ops.size() == 1 || Ops.size() == 3 || Ops.size() == 4) && in EmitX86CvtF16ToFloatExpr()
13869 if (Ops.size() == 4 && cast<llvm::ConstantInt>(Ops[3])->getZExtValue() != 4) { in EmitX86CvtF16ToFloatExpr()
13872 return CGF.Builder.CreateCall(F, {Ops[0], Ops[1], Ops[2], Ops[3]}); in EmitX86CvtF16ToFloatExpr()
13876 Value *Src = Ops[0]; in EmitX86CvtF16ToFloatExpr()
13893 if (Ops.size() >= 3) in EmitX86CvtF16ToFloatExpr()
13894 Res = EmitX86Select(CGF, Ops[2], Res, Ops[1]); in EmitX86CvtF16ToFloatExpr()
14071 SmallVector<Value*, 4> Ops; in EmitX86BuiltinExpr() local
14082 Ops.push_back(EmitScalarOrConstFoldImmArg(ICEArguments, i, E)); in EmitX86BuiltinExpr()
14091 auto getCmpIntrinsicCall = [this, &Ops](Intrinsic::ID ID, unsigned Imm) { in EmitX86BuiltinExpr()
14092 Ops.push_back(llvm::ConstantInt::get(Int8Ty, Imm)); in EmitX86BuiltinExpr()
14094 return Builder.CreateCall(F, Ops); in EmitX86BuiltinExpr()
14102 auto getVectorFCmpIR = [this, &Ops, E](CmpInst::Predicate Pred, in EmitX86BuiltinExpr()
14107 Cmp = Builder.CreateFCmpS(Pred, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
14109 Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
14110 llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
14119 Value *Address = Ops[0]; in EmitX86BuiltinExpr()
14120 ConstantInt *C = cast<ConstantInt>(Ops[1]); in EmitX86BuiltinExpr()
14129 Ops[0]); in EmitX86BuiltinExpr()
14149 Ops[0]); in EmitX86BuiltinExpr()
14155 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType()); in EmitX86BuiltinExpr()
14156 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)}); in EmitX86BuiltinExpr()
14161 Function *F = CGM.getIntrinsic(Intrinsic::cttz, Ops[0]->getType()); in EmitX86BuiltinExpr()
14162 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)}); in EmitX86BuiltinExpr()
14176 return Builder.CreateBitCast(BuildVector(Ops), in EmitX86BuiltinExpr()
14189 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
14190 uint64_t Index = cast<ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
14194 return Builder.CreateExtractElement(Ops[0], Index); in EmitX86BuiltinExpr()
14205 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
14206 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
14210 return Builder.CreateInsertElement(Ops[0], Ops[1], Index); in EmitX86BuiltinExpr()
14215 Builder.CreateStore(Ops[0], Tmp); in EmitX86BuiltinExpr()
14266 Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty); in EmitX86BuiltinExpr()
14267 Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty); in EmitX86BuiltinExpr()
14268 Ops[1] = Mhi; in EmitX86BuiltinExpr()
14269 Ops.push_back(Mlo); in EmitX86BuiltinExpr()
14270 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
14274 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_xgetbv), Ops); in EmitX86BuiltinExpr()
14293 return EmitX86MaskedStore(*this, Ops, Align(1)); in EmitX86BuiltinExpr()
14298 return EmitX86MaskedStore(*this, Ops, Align(1)); in EmitX86BuiltinExpr()
14314 return Builder.CreateCall(F, Ops); in EmitX86BuiltinExpr()
14328 return EmitX86SExtMask(*this, Ops[0], ConvertType(E->getType())); in EmitX86BuiltinExpr()
14342 return EmitX86ConvertToMask(*this, Ops[0]); in EmitX86BuiltinExpr()
14350 return EmitX86ConvertIntToFp(*this, E, Ops, /*IsSigned*/ true); in EmitX86BuiltinExpr()
14357 return EmitX86ConvertIntToFp(*this, E, Ops, /*IsSigned*/ false); in EmitX86BuiltinExpr()
14364 return EmitScalarFMAExpr(*this, E, Ops, Ops[0]); in EmitX86BuiltinExpr()
14367 return EmitScalarFMAExpr(*this, E, Ops, in EmitX86BuiltinExpr()
14368 Constant::getNullValue(Ops[0]->getType())); in EmitX86BuiltinExpr()
14372 return EmitScalarFMAExpr(*this, E, Ops, Ops[0], /*ZeroMask*/ true); in EmitX86BuiltinExpr()
14376 return EmitScalarFMAExpr(*this, E, Ops, Ops[2], /*ZeroMask*/ false, 2); in EmitX86BuiltinExpr()
14380 return EmitScalarFMAExpr(*this, E, Ops, Ops[2], /*ZeroMask*/ false, 2, in EmitX86BuiltinExpr()
14400 return EmitX86FMAExpr(*this, E, Ops, BuiltinID, /*IsAddSub*/ false); in EmitX86BuiltinExpr()
14413 return EmitX86FMAExpr(*this, E, Ops, BuiltinID, /*IsAddSub*/ true); in EmitX86BuiltinExpr()
14428 *this, Ops, in EmitX86BuiltinExpr()
14449 return EmitX86MaskedLoad(*this, Ops, Align(1)); in EmitX86BuiltinExpr()
14454 return EmitX86MaskedLoad(*this, Ops, Align(1)); in EmitX86BuiltinExpr()
14469 *this, Ops, in EmitX86BuiltinExpr()
14490 return EmitX86ExpandLoad(*this, Ops); in EmitX86BuiltinExpr()
14510 return EmitX86CompressStore(*this, Ops); in EmitX86BuiltinExpr()
14530 return EmitX86CompressExpand(*this, Ops, /*IsCompress*/false); in EmitX86BuiltinExpr()
14550 return EmitX86CompressExpand(*this, Ops, /*IsCompress*/true); in EmitX86BuiltinExpr()
14654 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(), in EmitX86BuiltinExpr()
14655 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements()); in EmitX86BuiltinExpr()
14656 Ops[3] = getMaskVecValue(*this, Ops[3], MinElts); in EmitX86BuiltinExpr()
14658 return Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
14763 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements(), in EmitX86BuiltinExpr()
14764 cast<llvm::FixedVectorType>(Ops[3]->getType())->getNumElements()); in EmitX86BuiltinExpr()
14765 Ops[1] = getMaskVecValue(*this, Ops[1], MinElts); in EmitX86BuiltinExpr()
14767 return Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
14789 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
14791 unsigned Index = cast<ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
14800 Value *Res = Builder.CreateShuffleVector(Ops[0], ArrayRef(Indices, NumElts), in EmitX86BuiltinExpr()
14803 if (Ops.size() == 4) in EmitX86BuiltinExpr()
14804 Res = EmitX86Select(*this, Ops[3], Res, Ops[2]); in EmitX86BuiltinExpr()
14825 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
14827 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements(); in EmitX86BuiltinExpr()
14829 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
14839 Ops[1], ArrayRef(Indices, DstNumElts), "widen"); in EmitX86BuiltinExpr()
14848 return Builder.CreateShuffleVector(Ops[0], Op1, in EmitX86BuiltinExpr()
14853 Value *Res = Builder.CreateTrunc(Ops[0], Ops[1]->getType()); in EmitX86BuiltinExpr()
14854 return EmitX86Select(*this, Ops[2], Res, Ops[1]); in EmitX86BuiltinExpr()
14859 if (const auto *C = dyn_cast<Constant>(Ops[2])) in EmitX86BuiltinExpr()
14861 return Builder.CreateTrunc(Ops[0], Ops[1]->getType()); in EmitX86BuiltinExpr()
14878 return Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
14889 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
14890 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
14898 return Builder.CreateShuffleVector(Ops[0], Ops[1], in EmitX86BuiltinExpr()
14904 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
14905 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
14921 return Builder.CreateShuffleVector(Ops[0], ArrayRef(Indices, NumElts), in EmitX86BuiltinExpr()
14927 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
14928 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
14944 return Builder.CreateShuffleVector(Ops[0], ArrayRef(Indices, NumElts), in EmitX86BuiltinExpr()
14956 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
14957 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
14973 return Builder.CreateShuffleVector(Ops[0], ArrayRef(Indices, NumElts), in EmitX86BuiltinExpr()
14982 uint32_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
14983 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
15002 return Builder.CreateShuffleVector(Ops[0], Ops[1], in EmitX86BuiltinExpr()
15009 unsigned Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
15010 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
15019 return Builder.CreateShuffleVector(Ops[0], ArrayRef(Indices, NumElts), in EmitX86BuiltinExpr()
15025 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
15028 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
15040 Ops[1] = Ops[0]; in EmitX86BuiltinExpr()
15041 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
15055 return Builder.CreateShuffleVector(Ops[1], Ops[0], in EmitX86BuiltinExpr()
15065 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
15066 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
15075 return Builder.CreateShuffleVector(Ops[1], Ops[0], in EmitX86BuiltinExpr()
15086 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
15087 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
15103 return Builder.CreateShuffleVector(Ops[0], Ops[1], in EmitX86BuiltinExpr()
15111 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
15113 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
15125 OutOps[l] = llvm::ConstantAggregateZero::get(Ops[0]->getType()); in EmitX86BuiltinExpr()
15127 OutOps[l] = Ops[1]; in EmitX86BuiltinExpr()
15129 OutOps[l] = Ops[0]; in EmitX86BuiltinExpr()
15149 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
15150 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
15169 Value *Cast = Builder.CreateBitCast(Ops[0], VecTy, "cast"); in EmitX86BuiltinExpr()
15173 return Builder.CreateBitCast(SV, Ops[0]->getType(), "cast"); in EmitX86BuiltinExpr()
15178 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
15179 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
15198 Value *Cast = Builder.CreateBitCast(Ops[0], VecTy, "cast"); in EmitX86BuiltinExpr()
15208 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
15209 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
15212 return llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
15214 Value *In = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
15223 return Builder.CreateBitCast(SV, Ops[0]->getType()); in EmitX86BuiltinExpr()
15229 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
15230 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
15233 return llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
15235 Value *In = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
15244 return Builder.CreateBitCast(SV, Ops[0]->getType()); in EmitX86BuiltinExpr()
15253 Value *Ptr = Ops[0]; in EmitX86BuiltinExpr()
15254 Value *Src = Ops[1]; in EmitX86BuiltinExpr()
15288 return EmitX86FunnelShift(*this, Ops[0], Ops[0], Ops[1], false); in EmitX86BuiltinExpr()
15301 return EmitX86FunnelShift(*this, Ops[0], Ops[0], Ops[1], true); in EmitX86BuiltinExpr()
15326 return EmitX86Select(*this, Ops[0], Ops[1], Ops[2]); in EmitX86BuiltinExpr()
15331 Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0); in EmitX86BuiltinExpr()
15332 Value *B = Builder.CreateExtractElement(Ops[2], (uint64_t)0); in EmitX86BuiltinExpr()
15333 A = EmitX86ScalarSelect(*this, Ops[0], A, B); in EmitX86BuiltinExpr()
15334 return Builder.CreateInsertElement(Ops[1], A, (uint64_t)0); in EmitX86BuiltinExpr()
15348 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7; in EmitX86BuiltinExpr()
15349 return EmitX86MaskedCompare(*this, CC, true, Ops); in EmitX86BuiltinExpr()
15363 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7; in EmitX86BuiltinExpr()
15364 return EmitX86MaskedCompare(*this, CC, false, Ops); in EmitX86BuiltinExpr()
15370 return EmitX86vpcom(*this, Ops, true); in EmitX86BuiltinExpr()
15375 return EmitX86vpcom(*this, Ops, false); in EmitX86BuiltinExpr()
15381 Value *Or = EmitX86MaskLogic(*this, Instruction::Or, Ops); in EmitX86BuiltinExpr()
15382 Value *C = llvm::Constant::getAllOnesValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
15390 Value *Or = EmitX86MaskLogic(*this, Instruction::Or, Ops); in EmitX86BuiltinExpr()
15391 Value *C = llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
15433 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
15434 Value *LHS = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
15435 Value *RHS = getMaskVecValue(*this, Ops[1], NumElts); in EmitX86BuiltinExpr()
15461 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
15462 Value *LHS = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
15463 Value *RHS = getMaskVecValue(*this, Ops[1], NumElts); in EmitX86BuiltinExpr()
15466 return Builder.CreateBitCast(Res, Ops[0]->getType()); in EmitX86BuiltinExpr()
15472 return EmitX86MaskLogic(*this, Instruction::And, Ops); in EmitX86BuiltinExpr()
15477 return EmitX86MaskLogic(*this, Instruction::And, Ops, true); in EmitX86BuiltinExpr()
15482 return EmitX86MaskLogic(*this, Instruction::Or, Ops); in EmitX86BuiltinExpr()
15487 return EmitX86MaskLogic(*this, Instruction::Xor, Ops, true); in EmitX86BuiltinExpr()
15492 return EmitX86MaskLogic(*this, Instruction::Xor, Ops); in EmitX86BuiltinExpr()
15497 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
15498 Value *Res = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
15500 Ops[0]->getType()); in EmitX86BuiltinExpr()
15509 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
15510 Value *Res = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
15511 return Builder.CreateBitCast(Res, Ops[0]->getType()); in EmitX86BuiltinExpr()
15517 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
15518 Value *LHS = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
15519 Value *RHS = getMaskVecValue(*this, Ops[1], NumElts); in EmitX86BuiltinExpr()
15532 return Builder.CreateBitCast(Res, Ops[0]->getType()); in EmitX86BuiltinExpr()
15541 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType()); in EmitX86BuiltinExpr()
15542 return Builder.CreateCall(F, {Ops[0],Builder.getInt1(false)}); in EmitX86BuiltinExpr()
15546 Value *A = Builder.CreateExtractElement(Ops[0], (uint64_t)0); in EmitX86BuiltinExpr()
15557 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0); in EmitX86BuiltinExpr()
15562 unsigned CC = cast<llvm::ConstantInt>(Ops[4])->getZExtValue(); in EmitX86BuiltinExpr()
15581 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
15583 Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0); in EmitX86BuiltinExpr()
15594 Value *Src = Builder.CreateExtractElement(Ops[2], (uint64_t)0); in EmitX86BuiltinExpr()
15595 A = EmitX86ScalarSelect(*this, Ops[3], A, Src); in EmitX86BuiltinExpr()
15596 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0); in EmitX86BuiltinExpr()
15607 if (Ops.size() == 2) { in EmitX86BuiltinExpr()
15608 unsigned CC = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
15627 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
15633 Ops[0]->getType()); in EmitX86BuiltinExpr()
15634 return Builder.CreateConstrainedFPCall(F, Ops[0]); in EmitX86BuiltinExpr()
15636 Function *F = CGM.getIntrinsic(Intrinsic::sqrt, Ops[0]->getType()); in EmitX86BuiltinExpr()
15637 return Builder.CreateCall(F, Ops[0]); in EmitX86BuiltinExpr()
15644 return EmitX86Muldq(*this, /*IsSigned*/false, Ops); in EmitX86BuiltinExpr()
15649 return EmitX86Muldq(*this, /*IsSigned*/true, Ops); in EmitX86BuiltinExpr()
15657 return EmitX86Ternlog(*this, /*ZeroMask*/false, Ops); in EmitX86BuiltinExpr()
15665 return EmitX86Ternlog(*this, /*ZeroMask*/true, Ops); in EmitX86BuiltinExpr()
15676 return EmitX86FunnelShift(*this, Ops[0], Ops[1], Ops[2], false); in EmitX86BuiltinExpr()
15688 return EmitX86FunnelShift(*this, Ops[1], Ops[0], Ops[2], true); in EmitX86BuiltinExpr()
15699 return EmitX86FunnelShift(*this, Ops[0], Ops[1], Ops[2], false); in EmitX86BuiltinExpr()
15711 return EmitX86FunnelShift(*this, Ops[1], Ops[0], Ops[2], true); in EmitX86BuiltinExpr()
15720 CGM.getIntrinsic(Intrinsic::vector_reduce_fadd, Ops[1]->getType()); in EmitX86BuiltinExpr()
15723 return Builder.CreateCall(F, {Ops[0], Ops[1]}); in EmitX86BuiltinExpr()
15731 CGM.getIntrinsic(Intrinsic::vector_reduce_fmul, Ops[1]->getType()); in EmitX86BuiltinExpr()
15734 return Builder.CreateCall(F, {Ops[0], Ops[1]}); in EmitX86BuiltinExpr()
15742 CGM.getIntrinsic(Intrinsic::vector_reduce_fmax, Ops[0]->getType()); in EmitX86BuiltinExpr()
15745 return Builder.CreateCall(F, {Ops[0]}); in EmitX86BuiltinExpr()
15753 CGM.getIntrinsic(Intrinsic::vector_reduce_fmin, Ops[0]->getType()); in EmitX86BuiltinExpr()
15756 return Builder.CreateCall(F, {Ops[0]}); in EmitX86BuiltinExpr()
15763 Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast"); in EmitX86BuiltinExpr()
15765 return Builder.CreateCall(F, Ops, "pswapd"); in EmitX86BuiltinExpr()
15798 Ops[0]); in EmitX86BuiltinExpr()
15823 { Ops[0], Ops[1], Ops[2] }); in EmitX86BuiltinExpr()
15825 Ops[3]); in EmitX86BuiltinExpr()
15839 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
15840 Value *MaskIn = Ops[2]; in EmitX86BuiltinExpr()
15841 Ops.erase(&Ops[2]); in EmitX86BuiltinExpr()
15875 Value *Fpclass = Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
15886 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
15911 Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID), {Ops[0], Ops[1]}); in EmitX86BuiltinExpr()
15914 Builder.CreateDefaultAlignedStore(Result, Ops[2]); in EmitX86BuiltinExpr()
15918 return Builder.CreateDefaultAlignedStore(Result, Ops[3]); in EmitX86BuiltinExpr()
15938 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
15945 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
15946 Value *MaskIn = Ops[2]; in EmitX86BuiltinExpr()
15947 Ops.erase(&Ops[2]); in EmitX86BuiltinExpr()
15963 Value *Shufbit = Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
16014 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x1f; in EmitX86BuiltinExpr()
16103 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
16104 Ops[3] = getMaskVecValue(*this, Ops[3], NumElts); in EmitX86BuiltinExpr()
16105 Value *Cmp = Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
16109 return Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
16120 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
16123 Cmp = Builder.CreateFCmpS(Pred, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
16125 Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
16126 return EmitX86MaskedCompareResult(*this, Cmp, NumElts, Ops[3]); in EmitX86BuiltinExpr()
16173 return EmitX86CvtF16ToFloatExpr(*this, Ops, ConvertType(E->getType())); in EmitX86BuiltinExpr()
16178 Ops[2] = getMaskVecValue( in EmitX86BuiltinExpr()
16179 *this, Ops[2], in EmitX86BuiltinExpr()
16180 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements()); in EmitX86BuiltinExpr()
16182 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
16185 return Builder.CreateFPExt(Ops[0], Builder.getFloatTy()); in EmitX86BuiltinExpr()
16199 Value *Res = Builder.CreateCall(CGM.getIntrinsic(IID), Ops[0]); in EmitX86BuiltinExpr()
16200 return EmitX86Select(*this, Ops[2], Res, Ops[1]); in EmitX86BuiltinExpr()
16247 Value *LHS = Builder.CreateIntCast(Ops[0], Int64Ty, isSigned); in EmitX86BuiltinExpr()
16248 Value *RHS = Builder.CreateIntCast(Ops[1], Int64Ty, isSigned); in EmitX86BuiltinExpr()
16259 Value *LHS = Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned); in EmitX86BuiltinExpr()
16260 Value *RHS = Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned); in EmitX86BuiltinExpr()
16292 std::swap(Ops[0], Ops[1]); in EmitX86BuiltinExpr()
16293 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty); in EmitX86BuiltinExpr()
16294 return Builder.CreateCall(F, Ops); in EmitX86BuiltinExpr()
16311 return Builder.CreateMemSet(Ops[0], Ops[1], Ops[2], Align(1), true); in EmitX86BuiltinExpr()
16334 Ops[0], llvm::PointerType::get(getLLVMContext(), 257)); in EmitX86BuiltinExpr()
16346 Ops[0], llvm::PointerType::get(getLLVMContext(), 256)); in EmitX86BuiltinExpr()
16355 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), {Ops[0], Ops[1]}); in EmitX86BuiltinExpr()
16359 Value *Ptr = Builder.CreateConstGEP1_32(Int8Ty, Ops[2], i * 16); in EmitX86BuiltinExpr()
16369 Builder.CreateCall(CGM.getIntrinsic(IID), {Ops[0], Ops[1], Ops[2]}); in EmitX86BuiltinExpr()
16373 Value *Ptr = Builder.CreateConstGEP1_32(Int8Ty, Ops[3], i * 16); in EmitX86BuiltinExpr()
16406 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), {Ops[1], Ops[2]}); in EmitX86BuiltinExpr()
16419 Builder.CreateDefaultAlignedStore(Out, Ops[0]); in EmitX86BuiltinExpr()
16424 Builder.CreateDefaultAlignedStore(Zero, Ops[0]); in EmitX86BuiltinExpr()
16457 InOps[0] = Ops[2]; in EmitX86BuiltinExpr()
16459 Value *Ptr = Builder.CreateConstGEP1_32(Ty, Ops[1], i); in EmitX86BuiltinExpr()
16477 Value *Ptr = Builder.CreateConstGEP1_32(Extract->getType(), Ops[0], i); in EmitX86BuiltinExpr()
16486 Value *Ptr = Builder.CreateConstGEP1_32(Out->getType(), Ops[0], i); in EmitX86BuiltinExpr()
16501 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
16502 return EmitX86Select(*this, Ops[3], Call, Ops[0]); in EmitX86BuiltinExpr()
16510 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
16511 Value *And = Builder.CreateAnd(Ops[3], llvm::ConstantInt::get(Int8Ty, 1)); in EmitX86BuiltinExpr()
16512 return EmitX86Select(*this, And, Call, Ops[0]); in EmitX86BuiltinExpr()
16520 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
16522 return Builder.CreateShuffleVector(Call, Ops[2], Mask); in EmitX86BuiltinExpr()
16526 CGM.getIntrinsic(Intrinsic::prefetch, Ops[0]->getType()), in EmitX86BuiltinExpr()
16527 {Ops[0], llvm::ConstantInt::get(Int32Ty, 0), Ops[1], in EmitX86BuiltinExpr()
16571 SmallVector<Value *, 2> Ops; in EmitPPCBuiltinExpr() local
16572 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitPPCBuiltinExpr()
16573 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitPPCBuiltinExpr()
16576 Ops[0] = Builder.CreateGEP(Int8Ty, Ops[1], Ops[0]); in EmitPPCBuiltinExpr()
16577 Ops.pop_back(); in EmitPPCBuiltinExpr()
16623 return Builder.CreateCall(F, Ops, ""); in EmitPPCBuiltinExpr()
16639 SmallVector<Value *, 3> Ops; in EmitPPCBuiltinExpr() local
16640 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitPPCBuiltinExpr()
16641 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitPPCBuiltinExpr()
16642 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitPPCBuiltinExpr()
16645 Ops[1] = Builder.CreateGEP(Int8Ty, Ops[2], Ops[1]); in EmitPPCBuiltinExpr()
16646 Ops.pop_back(); in EmitPPCBuiltinExpr()
16686 return Builder.CreateCall(F, Ops, ""); in EmitPPCBuiltinExpr()
16943 SmallVector<Value *, 2> Ops; in EmitPPCBuiltinExpr() local
16948 Ops.push_back(Builder.CreateBitCast(Op0, V1I128Ty)); in EmitPPCBuiltinExpr()
16949 Ops.push_back(Builder.CreateBitCast(Op1, V1I128Ty)); in EmitPPCBuiltinExpr()
16953 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops, ""); in EmitPPCBuiltinExpr()
16959 SmallVector<Value *, 3> Ops; in EmitPPCBuiltinExpr() local
16965 Ops.push_back(Builder.CreateBitCast(Op0, V1I128Ty)); in EmitPPCBuiltinExpr()
16966 Ops.push_back(Builder.CreateBitCast(Op1, V1I128Ty)); in EmitPPCBuiltinExpr()
16967 Ops.push_back(Builder.CreateBitCast(Op2, V1I128Ty)); in EmitPPCBuiltinExpr()
16984 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops, ""); in EmitPPCBuiltinExpr()
17421 SmallVector<Value *, 4> Ops; in EmitPPCBuiltinExpr() local
17424 Ops.push_back(EmitArrayToPointerDecay(E->getArg(i)).getPointer()); in EmitPPCBuiltinExpr()
17426 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitPPCBuiltinExpr()
17445 Value *Ptr = Ops[0]; in EmitPPCBuiltinExpr()
17463 std::reverse(Ops.begin() + 1, Ops.end()); in EmitPPCBuiltinExpr()
17480 Ops[0] = Builder.CreateGEP(Int8Ty, Ops[1], Ops[0]); in EmitPPCBuiltinExpr()
17482 Ops[1] = Builder.CreateGEP(Int8Ty, Ops[2], Ops[1]); in EmitPPCBuiltinExpr()
17484 Ops.pop_back(); in EmitPPCBuiltinExpr()
17486 return Builder.CreateCall(F, Ops, ""); in EmitPPCBuiltinExpr()
17494 for (unsigned i=1; i<Ops.size(); i++) in EmitPPCBuiltinExpr()
17495 CallOps.push_back(Ops[i]); in EmitPPCBuiltinExpr()
17498 return Builder.CreateAlignedStore(Call, Ops[0], MaybeAlign(64)); in EmitPPCBuiltinExpr()
20635 Value *Ops[18]; in EmitWebAssemblyBuiltinExpr() local
20637 Ops[OpIdx++] = EmitScalarExpr(E->getArg(0)); in EmitWebAssemblyBuiltinExpr()
20638 Ops[OpIdx++] = EmitScalarExpr(E->getArg(1)); in EmitWebAssemblyBuiltinExpr()
20643 Ops[OpIdx++] = llvm::ConstantInt::get(getLLVMContext(), *LaneConst); in EmitWebAssemblyBuiltinExpr()
20646 return Builder.CreateCall(Callee, Ops); in EmitWebAssemblyBuiltinExpr()
20926 SmallVector<llvm::Value*,5> Ops = { Base }; in EmitHexagonBuiltinExpr() local
20928 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitHexagonBuiltinExpr()
20930 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(IntID), Ops); in EmitHexagonBuiltinExpr()
21041 SmallVector<llvm::Value*,4> Ops; in EmitHexagonBuiltinExpr() local
21047 Ops.push_back(V2Q(EmitScalarExpr(PredOp))); in EmitHexagonBuiltinExpr()
21050 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitHexagonBuiltinExpr()
21051 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitHexagonBuiltinExpr()
21098 SmallVector<Value *, 4> Ops; in EmitRISCVBuiltinExpr() local
21127 Ops.push_back(AggValue); in EmitRISCVBuiltinExpr()
21130 Ops.push_back(EmitScalarOrConstFoldImmArg(ICEArguments, i, E)); in EmitRISCVBuiltinExpr()
21175 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType()); in EmitRISCVBuiltinExpr()
21176 Value *Result = Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)}); in EmitRISCVBuiltinExpr()
21184 Function *F = CGM.getIntrinsic(Intrinsic::cttz, Ops[0]->getType()); in EmitRISCVBuiltinExpr()
21185 Value *Result = Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)}); in EmitRISCVBuiltinExpr()
21269 if (Ops.size() == 2) in EmitRISCVBuiltinExpr()
21270 DomainVal = cast<ConstantInt>(Ops[1])->getZExtValue(); in EmitRISCVBuiltinExpr()
21287 Address(Ops[0], ResTy, CharUnits::fromQuantity(Width / 8))); in EmitRISCVBuiltinExpr()
21297 if (Ops.size() == 3) in EmitRISCVBuiltinExpr()
21298 DomainVal = cast<ConstantInt>(Ops[2])->getZExtValue(); in EmitRISCVBuiltinExpr()
21306 StoreInst *Store = Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitRISCVBuiltinExpr()
21323 return Builder.CreateCall(F, Ops, ""); in EmitRISCVBuiltinExpr()