Lines Matching refs:Ops
5522 Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops, in EmitNeonCall() argument
5532 Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift); in EmitNeonCall()
5534 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name); in EmitNeonCall()
5538 return Builder.CreateConstrainedFPCall(F, Ops, name); in EmitNeonCall()
5540 return Builder.CreateCall(F, Ops, name); in EmitNeonCall()
6486 SmallVectorImpl<Value *> &Ops, const CallExpr *E) { in EmitCommonNeonSISDBuiltinExpr() argument
6508 std::swap(Ops[0], Ops[1]); in EmitCommonNeonSISDBuiltinExpr()
6524 if (Ops[j]->getType()->getPrimitiveSizeInBits() == in EmitCommonNeonSISDBuiltinExpr()
6528 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy()); in EmitCommonNeonSISDBuiltinExpr()
6531 Ops[j] = CGF.Builder.CreateTruncOrBitCast( in EmitCommonNeonSISDBuiltinExpr()
6532 Ops[j], cast<llvm::VectorType>(ArgTy)->getElementType()); in EmitCommonNeonSISDBuiltinExpr()
6533 Ops[j] = in EmitCommonNeonSISDBuiltinExpr()
6534 CGF.Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0); in EmitCommonNeonSISDBuiltinExpr()
6537 Value *Result = CGF.EmitNeonCall(F, Ops, s); in EmitCommonNeonSISDBuiltinExpr()
6549 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1, in EmitCommonNeonBuiltinExpr() argument
6592 Ops[0] = Builder.CreateBitCast(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
6593 return EmitNeonSplat(Ops[0], cast<ConstantInt>(Ops[1]), NumElements); in EmitCommonNeonBuiltinExpr()
6605 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs"); in EmitCommonNeonBuiltinExpr()
6606 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs"); in EmitCommonNeonBuiltinExpr()
6610 Ops[0] = Builder.CreateBitCast(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
6611 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); in EmitCommonNeonBuiltinExpr()
6612 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
6613 return Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
6620 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
6621 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); in EmitCommonNeonBuiltinExpr()
6622 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn"); in EmitCommonNeonBuiltinExpr()
6627 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn"); in EmitCommonNeonBuiltinExpr()
6630 return Builder.CreateTrunc(Ops[0], VTy, "vaddhn"); in EmitCommonNeonBuiltinExpr()
6636 std::swap(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
6658 return EmitNeonCall(F, Ops, NameHint); in EmitCommonNeonBuiltinExpr()
6662 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ, in EmitCommonNeonBuiltinExpr()
6666 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE, in EmitCommonNeonBuiltinExpr()
6670 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE, in EmitCommonNeonBuiltinExpr()
6674 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT, in EmitCommonNeonBuiltinExpr()
6678 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT, in EmitCommonNeonBuiltinExpr()
6684 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef())); in EmitCommonNeonBuiltinExpr()
6688 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
6691 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") in EmitCommonNeonBuiltinExpr()
6692 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); in EmitCommonNeonBuiltinExpr()
6695 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
6698 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") in EmitCommonNeonBuiltinExpr()
6699 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); in EmitCommonNeonBuiltinExpr()
6709 return EmitNeonCall(F, Ops, "vcvt_n"); in EmitCommonNeonBuiltinExpr()
6725 return EmitNeonCall(F, Ops, "vcvt_n"); in EmitCommonNeonBuiltinExpr()
6739 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); in EmitCommonNeonBuiltinExpr()
6740 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt") in EmitCommonNeonBuiltinExpr()
6741 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt"); in EmitCommonNeonBuiltinExpr()
6792 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
6796 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
6801 int CV = cast<ConstantInt>(Ops[2])->getSExtValue(); in EmitCommonNeonBuiltinExpr()
6806 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
6807 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
6808 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices, "vext"); in EmitCommonNeonBuiltinExpr()
6812 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
6813 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
6814 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
6819 {Ops[1], Ops[2], Ops[0]}); in EmitCommonNeonBuiltinExpr()
6824 Ops.push_back(getAlignmentValue32(PtrOp0)); in EmitCommonNeonBuiltinExpr()
6825 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1"); in EmitCommonNeonBuiltinExpr()
6834 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitCommonNeonBuiltinExpr()
6837 Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN"); in EmitCommonNeonBuiltinExpr()
6838 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitCommonNeonBuiltinExpr()
6839 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
6840 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitCommonNeonBuiltinExpr()
6857 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint); in EmitCommonNeonBuiltinExpr()
6858 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitCommonNeonBuiltinExpr()
6859 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
6860 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitCommonNeonBuiltinExpr()
6868 Ops[0] = Builder.CreateInsertElement(V, Ld, CI); in EmitCommonNeonBuiltinExpr()
6869 return EmitNeonSplat(Ops[0], CI); in EmitCommonNeonBuiltinExpr()
6879 for (unsigned I = 2; I < Ops.size() - 1; ++I) in EmitCommonNeonBuiltinExpr()
6880 Ops[I] = Builder.CreateBitCast(Ops[I], Ty); in EmitCommonNeonBuiltinExpr()
6881 Ops.push_back(getAlignmentValue32(PtrOp1)); in EmitCommonNeonBuiltinExpr()
6882 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint); in EmitCommonNeonBuiltinExpr()
6883 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitCommonNeonBuiltinExpr()
6884 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
6885 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitCommonNeonBuiltinExpr()
6890 Ops[0] = Builder.CreateBitCast(Ops[0], DTy); in EmitCommonNeonBuiltinExpr()
6892 return Builder.CreateZExt(Ops[0], Ty, "vmovl"); in EmitCommonNeonBuiltinExpr()
6893 return Builder.CreateSExt(Ops[0], Ty, "vmovl"); in EmitCommonNeonBuiltinExpr()
6898 Ops[0] = Builder.CreateBitCast(Ops[0], QTy); in EmitCommonNeonBuiltinExpr()
6899 return Builder.CreateTrunc(Ops[0], Ty, "vmovn"); in EmitCommonNeonBuiltinExpr()
6909 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); in EmitCommonNeonBuiltinExpr()
6919 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
6929 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl"); in EmitCommonNeonBuiltinExpr()
6933 SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end()); in EmitCommonNeonBuiltinExpr()
6934 Ops[1] = in EmitCommonNeonBuiltinExpr()
6936 Ops.resize(2); in EmitCommonNeonBuiltinExpr()
6937 return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
6951 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
6960 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
6964 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n", in EmitCommonNeonBuiltinExpr()
6968 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n", in EmitCommonNeonBuiltinExpr()
6975 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
6981 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
6984 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", in EmitCommonNeonBuiltinExpr()
6991 return EmitNeonCall(F, Ops, ""); in EmitCommonNeonBuiltinExpr()
6995 Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false); in EmitCommonNeonBuiltinExpr()
6996 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1], in EmitCommonNeonBuiltinExpr()
7001 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
7003 Ops[0] = Builder.CreateZExt(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
7005 Ops[0] = Builder.CreateSExt(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
7006 Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false); in EmitCommonNeonBuiltinExpr()
7007 return Builder.CreateShl(Ops[0], Ops[1], "vshll_n"); in EmitCommonNeonBuiltinExpr()
7012 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
7013 Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false); in EmitCommonNeonBuiltinExpr()
7015 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
7017 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
7018 return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n"); in EmitCommonNeonBuiltinExpr()
7022 return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n"); in EmitCommonNeonBuiltinExpr()
7038 Ops.push_back(getAlignmentValue32(PtrOp0)); in EmitCommonNeonBuiltinExpr()
7039 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, ""); in EmitCommonNeonBuiltinExpr()
7047 return EmitNeonCall(F, Ops, ""); in EmitCommonNeonBuiltinExpr()
7054 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); in EmitCommonNeonBuiltinExpr()
7055 return EmitNeonCall(F, Ops, ""); in EmitCommonNeonBuiltinExpr()
7069 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitCommonNeonBuiltinExpr()
7070 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, ""); in EmitCommonNeonBuiltinExpr()
7073 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, ""); in EmitCommonNeonBuiltinExpr()
7080 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
7081 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); in EmitCommonNeonBuiltinExpr()
7082 Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn"); in EmitCommonNeonBuiltinExpr()
7087 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn"); in EmitCommonNeonBuiltinExpr()
7090 return Builder.CreateTrunc(Ops[0], VTy, "vsubhn"); in EmitCommonNeonBuiltinExpr()
7094 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitCommonNeonBuiltinExpr()
7095 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
7096 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
7105 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
7106 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn"); in EmitCommonNeonBuiltinExpr()
7113 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
7114 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
7115 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
7116 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], in EmitCommonNeonBuiltinExpr()
7118 return Builder.CreateSExt(Ops[0], Ty, "vtst"); in EmitCommonNeonBuiltinExpr()
7122 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitCommonNeonBuiltinExpr()
7123 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
7124 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
7132 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
7133 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp"); in EmitCommonNeonBuiltinExpr()
7140 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty); in EmitCommonNeonBuiltinExpr()
7141 return EmitNeonCall(F, Ops, ""); in EmitCommonNeonBuiltinExpr()
7145 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitCommonNeonBuiltinExpr()
7146 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
7147 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
7156 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
7157 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip"); in EmitCommonNeonBuiltinExpr()
7168 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vdot"); in EmitCommonNeonBuiltinExpr()
7175 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlal_low"); in EmitCommonNeonBuiltinExpr()
7182 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlsl_low"); in EmitCommonNeonBuiltinExpr()
7189 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlal_high"); in EmitCommonNeonBuiltinExpr()
7196 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlsl_high"); in EmitCommonNeonBuiltinExpr()
7203 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmmla"); in EmitCommonNeonBuiltinExpr()
7209 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vusmmla"); in EmitCommonNeonBuiltinExpr()
7216 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vusdot"); in EmitCommonNeonBuiltinExpr()
7223 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vbfdot"); in EmitCommonNeonBuiltinExpr()
7228 return EmitNeonCall(F, Ops, "vcvtfp2bf"); in EmitCommonNeonBuiltinExpr()
7238 Value *Result = EmitNeonCall(F, Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7269 static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops, in packTBLDVectorList() argument
7279 auto *TblTy = cast<llvm::FixedVectorType>(Ops[0]->getType()); in packTBLDVectorList()
7285 int PairPos = 0, End = Ops.size() - 1; in packTBLDVectorList()
7287 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], in packTBLDVectorList()
7288 Ops[PairPos+1], Indices, in packTBLDVectorList()
7297 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], in packTBLDVectorList()
7370 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) }; in EmitSpecialRegisterBuiltin() local
7371 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitSpecialRegisterBuiltin()
7523 Value *Ops[2]; in EmitARMBuiltinExpr() local
7525 Ops[i] = EmitScalarExpr(E->getArg(i)); in EmitARMBuiltinExpr()
7529 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); in EmitARMBuiltinExpr()
7828 SmallVector<Value*, 4> Ops; in EmitARMBuiltinExpr() local
7859 Ops.push_back(PtrOp0.getPointer()); in EmitARMBuiltinExpr()
7886 Ops.push_back(PtrOp1.getPointer()); in EmitARMBuiltinExpr()
7892 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitARMBuiltinExpr()
7896 Ops.push_back(llvm::ConstantInt::get( in EmitARMBuiltinExpr()
7919 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane"); in EmitARMBuiltinExpr()
7939 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitARMBuiltinExpr()
7942 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops, in EmitARMBuiltinExpr()
7945 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops, in EmitARMBuiltinExpr()
7948 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops, in EmitARMBuiltinExpr()
7951 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops, in EmitARMBuiltinExpr()
7955 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vcvtbfp2bf), Ops, in EmitARMBuiltinExpr()
7966 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0], in EmitARMBuiltinExpr()
7967 Ops[3], Ops[4], Ops[5]}); in EmitARMBuiltinExpr()
7993 return Builder.CreateCall(F, Ops, "vcvtr"); in EmitARMBuiltinExpr()
8016 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch); in EmitARMBuiltinExpr()
8026 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
8027 int Lane = cast<ConstantInt>(Ops[2])->getZExtValue(); in EmitARMBuiltinExpr()
8029 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); in EmitARMBuiltinExpr()
8035 Value *Ld = Builder.CreateCall(F, {Ops[0], Align}); in EmitARMBuiltinExpr()
8038 return Builder.CreateShuffleVector(Ops[1], Ld, Indices, "vld1q_lane"); in EmitARMBuiltinExpr()
8042 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
8045 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane"); in EmitARMBuiltinExpr()
8050 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n", in EmitARMBuiltinExpr()
8054 Ops, "vqrshrun_n", 1, true); in EmitARMBuiltinExpr()
8057 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n", in EmitARMBuiltinExpr()
8061 Ops, "vqshrun_n", 1, true); in EmitARMBuiltinExpr()
8065 Ops, "vrecpe"); in EmitARMBuiltinExpr()
8068 Ops, "vrshrn_n", 1, true); in EmitARMBuiltinExpr()
8071 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitARMBuiltinExpr()
8072 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
8073 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true); in EmitARMBuiltinExpr()
8075 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]}); in EmitARMBuiltinExpr()
8076 return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n"); in EmitARMBuiltinExpr()
8083 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift); in EmitARMBuiltinExpr()
8085 Ops, "vsli_n"); in EmitARMBuiltinExpr()
8088 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitARMBuiltinExpr()
8089 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); in EmitARMBuiltinExpr()
8090 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitARMBuiltinExpr()
8095 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
8096 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2])); in EmitARMBuiltinExpr()
8097 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); in EmitARMBuiltinExpr()
8098 Ops[2] = getAlignmentValue32(PtrOp0); in EmitARMBuiltinExpr()
8099 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()}; in EmitARMBuiltinExpr()
8101 Tys), Ops); in EmitARMBuiltinExpr()
8105 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
8106 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); in EmitARMBuiltinExpr()
8108 Ops[1], Builder.CreateElementBitCast(PtrOp0, Ops[1]->getType())); in EmitARMBuiltinExpr()
8113 Ops, "vtbl1"); in EmitARMBuiltinExpr()
8116 Ops, "vtbl2"); in EmitARMBuiltinExpr()
8119 Ops, "vtbl3"); in EmitARMBuiltinExpr()
8122 Ops, "vtbl4"); in EmitARMBuiltinExpr()
8125 Ops, "vtbx1"); in EmitARMBuiltinExpr()
8128 Ops, "vtbx2"); in EmitARMBuiltinExpr()
8131 Ops, "vtbx3"); in EmitARMBuiltinExpr()
8134 Ops, "vtbx4"); in EmitARMBuiltinExpr()
8282 llvm::SmallVector<Value *, 4> Ops; in EmitARMMVEBuiltinExpr() local
8301 Ops.push_back(EmitScalarExpr(Addr)); in EmitARMMVEBuiltinExpr()
8305 Value *LoadResult = Builder.CreateCall(F, Ops); in EmitARMMVEBuiltinExpr()
8319 llvm::SmallVector<Value *, 4> Ops; in EmitARMMVEBuiltinExpr() local
8323 Ops.push_back(EmitScalarExpr(Addr)); in EmitARMMVEBuiltinExpr()
8344 Ops.push_back(Builder.CreateExtractValue(Mvec, {0, i})); in EmitARMMVEBuiltinExpr()
8349 Ops.push_back(llvm::ConstantInt::get(Int32Ty, i)); in EmitARMMVEBuiltinExpr()
8350 ToReturn = Builder.CreateCall(F, Ops); in EmitARMMVEBuiltinExpr()
8351 Ops.pop_back(); in EmitARMMVEBuiltinExpr()
8372 SmallVectorImpl<Value *> &Ops, in EmitAArch64TblBuiltinExpr() argument
8428 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 1), nullptr, in EmitAArch64TblBuiltinExpr()
8429 Ops[1], Ty, Intrinsic::aarch64_neon_tbl1, in EmitAArch64TblBuiltinExpr()
8433 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 2), nullptr, in EmitAArch64TblBuiltinExpr()
8434 Ops[2], Ty, Intrinsic::aarch64_neon_tbl1, in EmitAArch64TblBuiltinExpr()
8438 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 3), nullptr, in EmitAArch64TblBuiltinExpr()
8439 Ops[3], Ty, Intrinsic::aarch64_neon_tbl2, in EmitAArch64TblBuiltinExpr()
8443 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 4), nullptr, in EmitAArch64TblBuiltinExpr()
8444 Ops[4], Ty, Intrinsic::aarch64_neon_tbl2, in EmitAArch64TblBuiltinExpr()
8449 packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 1), nullptr, Ops[2], in EmitAArch64TblBuiltinExpr()
8453 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV); in EmitAArch64TblBuiltinExpr()
8456 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); in EmitAArch64TblBuiltinExpr()
8461 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 2), Ops[0], in EmitAArch64TblBuiltinExpr()
8462 Ops[3], Ty, Intrinsic::aarch64_neon_tbx1, in EmitAArch64TblBuiltinExpr()
8467 packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 3), nullptr, Ops[4], in EmitAArch64TblBuiltinExpr()
8471 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4], in EmitAArch64TblBuiltinExpr()
8475 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); in EmitAArch64TblBuiltinExpr()
8480 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 4), Ops[0], in EmitAArch64TblBuiltinExpr()
8481 Ops[5], Ty, Intrinsic::aarch64_neon_tbx2, in EmitAArch64TblBuiltinExpr()
8515 return CGF.EmitNeonCall(F, Ops, s); in EmitAArch64TblBuiltinExpr()
8696 SmallVectorImpl<Value *> &Ops, in EmitSVEGatherLoad() argument
8707 Ops[0] = EmitSVEPredicateCast(Ops[0], OverloadedTy); in EmitSVEGatherLoad()
8710 if (Ops[1]->getType()->isVectorTy()) in EmitSVEGatherLoad()
8714 F = CGM.getIntrinsic(IntID, {OverloadedTy, Ops[1]->getType()}); in EmitSVEGatherLoad()
8725 if (Ops.size() == 2) { in EmitSVEGatherLoad()
8726 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset"); in EmitSVEGatherLoad()
8727 Ops.push_back(ConstantInt::get(Int64Ty, 0)); in EmitSVEGatherLoad()
8732 if (!TypeFlags.isByteIndexed() && Ops[1]->getType()->isVectorTy()) { in EmitSVEGatherLoad()
8736 Ops[2] = Builder.CreateMul(Ops[2], Scale); in EmitSVEGatherLoad()
8739 Value *Call = Builder.CreateCall(F, Ops); in EmitSVEGatherLoad()
8748 SmallVectorImpl<Value *> &Ops, in EmitSVEScatterStore() argument
8756 Ops.insert(Ops.begin(), Ops.pop_back_val()); in EmitSVEScatterStore()
8759 if (Ops[2]->getType()->isVectorTy()) in EmitSVEScatterStore()
8763 F = CGM.getIntrinsic(IntID, {OverloadedTy, Ops[2]->getType()}); in EmitSVEScatterStore()
8774 if (Ops.size() == 3) { in EmitSVEScatterStore()
8775 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset"); in EmitSVEScatterStore()
8776 Ops.push_back(ConstantInt::get(Int64Ty, 0)); in EmitSVEScatterStore()
8781 Ops[0] = Builder.CreateTrunc(Ops[0], OverloadedTy); in EmitSVEScatterStore()
8788 Ops[1] = EmitSVEPredicateCast(Ops[1], OverloadedTy); in EmitSVEScatterStore()
8792 if (!TypeFlags.isByteIndexed() && Ops[2]->getType()->isVectorTy()) { in EmitSVEScatterStore()
8796 Ops[3] = Builder.CreateMul(Ops[3], Scale); in EmitSVEScatterStore()
8799 return Builder.CreateCall(F, Ops); in EmitSVEScatterStore()
8803 SmallVectorImpl<Value *> &Ops, in EmitSVEGatherPrefetch() argument
8807 auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Ops[1]->getType()); in EmitSVEGatherPrefetch()
8809 OverloadedTy = cast<llvm::ScalableVectorType>(Ops[2]->getType()); in EmitSVEGatherPrefetch()
8812 Ops[0] = EmitSVEPredicateCast(Ops[0], OverloadedTy); in EmitSVEGatherPrefetch()
8815 if (Ops[1]->getType()->isVectorTy()) { in EmitSVEGatherPrefetch()
8816 if (Ops.size() == 3) { in EmitSVEGatherPrefetch()
8818 Ops.push_back(ConstantInt::get(Int64Ty, 0)); in EmitSVEGatherPrefetch()
8821 std::swap(Ops[2], Ops[3]); in EmitSVEGatherPrefetch()
8827 Ops[2] = Builder.CreateMul(Ops[2], Scale); in EmitSVEGatherPrefetch()
8832 return Builder.CreateCall(F, Ops); in EmitSVEGatherPrefetch()
8836 SmallVectorImpl<Value*> &Ops, in EmitSVEStructLoad() argument
8859 Value *Predicate = EmitSVEPredicateCast(Ops[0], VTy); in EmitSVEStructLoad()
8860 Value *BasePtr= Builder.CreateBitCast(Ops[1], VecPtrTy); in EmitSVEStructLoad()
8861 Value *Offset = Ops.size() > 2 ? Ops[2] : Builder.getInt32(0); in EmitSVEStructLoad()
8870 SmallVectorImpl<Value*> &Ops, in EmitSVEStructStore() argument
8893 Value *Predicate = EmitSVEPredicateCast(Ops[0], VTy); in EmitSVEStructStore()
8894 Value *BasePtr = Builder.CreateBitCast(Ops[1], VecPtrTy); in EmitSVEStructStore()
8895 Value *Offset = Ops.size() > 3 ? Ops[2] : Builder.getInt32(0); in EmitSVEStructStore()
8896 Value *Val = Ops.back(); in EmitSVEStructStore()
8917 SmallVectorImpl<Value *> &Ops, in EmitSVEPMull() argument
8922 Ops[OpNo] = EmitSVEDupX(Ops[OpNo]); in EmitSVEPMull()
8926 Function *F = CGM.getIntrinsic(BuiltinID, Ops[0]->getType()); in EmitSVEPMull()
8927 Value *Call = Builder.CreateCall(F, {Ops[0], Ops[1]}); in EmitSVEPMull()
8935 ArrayRef<Value *> Ops, unsigned BuiltinID) { in EmitSVEMovl() argument
8938 return Builder.CreateCall(F, {Ops[0], Builder.getInt32(0)}); in EmitSVEMovl()
8942 SmallVectorImpl<Value *> &Ops, in EmitSVEPrefetchLoad() argument
8948 Value *Predicate = EmitSVEPredicateCast(Ops[0], MemoryTy); in EmitSVEPrefetchLoad()
8949 Value *BasePtr = Ops[1]; in EmitSVEPrefetchLoad()
8952 if (Ops.size() > 3) { in EmitSVEPrefetchLoad()
8954 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]); in EmitSVEPrefetchLoad()
8959 Value *PrfOp = Ops.back(); in EmitSVEPrefetchLoad()
8967 SmallVectorImpl<Value *> &Ops, in EmitSVEMaskedLoad() argument
8979 Value *Predicate = EmitSVEPredicateCast(Ops[0], MemoryTy); in EmitSVEMaskedLoad()
8980 Value *BasePtr = Builder.CreateBitCast(Ops[1], MemoryTy->getPointerTo()); in EmitSVEMaskedLoad()
8981 Value *Offset = Ops.size() > 2 ? Ops[2] : Builder.getInt32(0); in EmitSVEMaskedLoad()
8996 SmallVectorImpl<Value *> &Ops, in EmitSVEMaskedStore() argument
9004 auto VectorTy = cast<llvm::ScalableVectorType>(Ops.back()->getType()); in EmitSVEMaskedStore()
9007 Value *Predicate = EmitSVEPredicateCast(Ops[0], MemoryTy); in EmitSVEMaskedStore()
9008 Value *BasePtr = Builder.CreateBitCast(Ops[1], MemoryTy->getPointerTo()); in EmitSVEMaskedStore()
9009 Value *Offset = Ops.size() == 4 ? Ops[2] : Builder.getInt32(0); in EmitSVEMaskedStore()
9013 llvm::Value *Val = Builder.CreateTrunc(Ops.back(), MemoryTy); in EmitSVEMaskedStore()
9046 SmallVectorImpl<Value *> &Ops) { in InsertExplicitZeroOperand() argument
9048 Ops.insert(Ops.begin(), SplatZero); in InsertExplicitZeroOperand()
9052 SmallVectorImpl<Value *> &Ops) { in InsertExplicitUndefOperand() argument
9054 Ops.insert(Ops.begin(), SplatUndef); in InsertExplicitUndefOperand()
9060 ArrayRef<Value *> Ops) { in getSVEOverloadTypes() argument
9067 return {DefaultType, Ops[1]->getType()}; in getSVEOverloadTypes()
9070 return {getSVEPredType(TypeFlags), Ops[0]->getType()}; in getSVEOverloadTypes()
9073 return {Ops[0]->getType(), Ops.back()->getType()}; in getSVEOverloadTypes()
9076 return {ResultType, Ops[0]->getType()}; in getSVEOverloadTypes()
9097 llvm::SmallVector<Value *, 4> Ops; in EmitAArch64SVEBuiltinExpr() local
9100 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitAArch64SVEBuiltinExpr()
9112 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), *Result)); in EmitAArch64SVEBuiltinExpr()
9120 return EmitSVEMaskedLoad(E, Ty, Ops, Builtin->LLVMIntrinsic, in EmitAArch64SVEBuiltinExpr()
9123 return EmitSVEMaskedStore(E, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
9125 return EmitSVEGatherLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
9127 return EmitSVEScatterStore(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
9129 return EmitSVEPrefetchLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
9131 return EmitSVEGatherPrefetch(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
9133 return EmitSVEStructLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
9135 return EmitSVEStructStore(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
9140 InsertExplicitZeroOperand(Builder, Ty, Ops); in EmitAArch64SVEBuiltinExpr()
9143 InsertExplicitUndefOperand(Builder, Ty, Ops); in EmitAArch64SVEBuiltinExpr()
9148 Ops.push_back(Builder.getInt32(/*SV_ALL*/ 31)); in EmitAArch64SVEBuiltinExpr()
9150 Ops.insert(&Ops[1], Builder.getInt32(/*SV_ALL*/ 31)); in EmitAArch64SVEBuiltinExpr()
9153 for (unsigned i = 0, e = Ops.size(); i != e; ++i) in EmitAArch64SVEBuiltinExpr()
9154 if (auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType())) in EmitAArch64SVEBuiltinExpr()
9156 Ops[i] = EmitSVEPredicateCast(Ops[i], getSVEType(TypeFlags)); in EmitAArch64SVEBuiltinExpr()
9161 Ops[OpNo] = EmitSVEDupX(Ops[OpNo]); in EmitAArch64SVEBuiltinExpr()
9165 std::swap(Ops[1], Ops[2]); in EmitAArch64SVEBuiltinExpr()
9168 std::swap(Ops[1], Ops[2]); in EmitAArch64SVEBuiltinExpr()
9172 llvm::Type *OpndTy = Ops[1]->getType(); in EmitAArch64SVEBuiltinExpr()
9175 Ops[1] = Builder.CreateCall(Sel, {Ops[0], Ops[1], SplatZero}); in EmitAArch64SVEBuiltinExpr()
9179 getSVEOverloadTypes(TypeFlags, Ty, Ops)); in EmitAArch64SVEBuiltinExpr()
9180 Value *Call = Builder.CreateCall(F, Ops); in EmitAArch64SVEBuiltinExpr()
9199 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[1]}); in EmitAArch64SVEBuiltinExpr()
9207 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[0]}); in EmitAArch64SVEBuiltinExpr()
9213 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllb); in EmitAArch64SVEBuiltinExpr()
9218 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllb); in EmitAArch64SVEBuiltinExpr()
9223 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllt); in EmitAArch64SVEBuiltinExpr()
9228 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllt); in EmitAArch64SVEBuiltinExpr()
9234 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullt_pair); in EmitAArch64SVEBuiltinExpr()
9240 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullb_pair); in EmitAArch64SVEBuiltinExpr()
9247 Builder.CreateICmpNE(Ops[0], Constant::getNullValue(Ops[0]->getType())); in EmitAArch64SVEBuiltinExpr()
9271 unsigned NumOpnds = Ops.size(); in EmitAArch64SVEBuiltinExpr()
9279 llvm::Type *EltTy = Ops[0]->getType(); in EmitAArch64SVEBuiltinExpr()
9285 VecOps.push_back(Builder.CreateZExt(Ops[I], EltTy)); in EmitAArch64SVEBuiltinExpr()
9353 Value *V0 = Builder.CreateCall(FExtr, {Ops[0], Builder.getInt32(0)}); in EmitAArch64SVEBuiltinExpr()
9354 Value *V1 = Builder.CreateCall(FExtr, {Ops[0], Builder.getInt32(1)}); in EmitAArch64SVEBuiltinExpr()
9356 return Builder.CreateCall(F, {V0, V1, Ops[1]}); in EmitAArch64SVEBuiltinExpr()
9371 return Builder.CreateInsertVector(Ty, Ops[0], Ops[1], Builder.getInt64(0)); in EmitAArch64SVEBuiltinExpr()
9386 return Builder.CreateExtractVector(Ty, Ops[0], Builder.getInt64(0)); in EmitAArch64SVEBuiltinExpr()
9401 Value *Insert = Builder.CreateInsertVector(Ty, UndefValue::get(Ty), Ops[0], in EmitAArch64SVEBuiltinExpr()
9605 Value *Ops[2]; in EmitAArch64BuiltinExpr() local
9607 Ops[i] = EmitScalarExpr(E->getArg(i)); in EmitAArch64BuiltinExpr()
9611 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); in EmitAArch64BuiltinExpr()
9727 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)}; in EmitAArch64BuiltinExpr() local
9728 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitAArch64BuiltinExpr()
9924 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) }; in EmitAArch64BuiltinExpr() local
9925 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitAArch64BuiltinExpr()
9986 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "x18")}; in EmitAArch64BuiltinExpr() local
9987 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitAArch64BuiltinExpr()
10011 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "x18")}; in EmitAArch64BuiltinExpr() local
10012 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitAArch64BuiltinExpr()
10039 llvm::SmallVector<Value*, 4> Ops; in EmitAArch64BuiltinExpr() local
10057 Ops.push_back(PtrOp0.getPointer()); in EmitAArch64BuiltinExpr()
10062 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitAArch64BuiltinExpr()
10066 Ops.push_back(llvm::ConstantInt::get( in EmitAArch64BuiltinExpr()
10077 Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1))); in EmitAArch64BuiltinExpr()
10078 Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E); in EmitAArch64BuiltinExpr()
10096 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10097 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, HalfTy), Ops, "vabs"); in EmitAArch64BuiltinExpr()
10100 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10101 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
10102 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
10103 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10105 return Builder.CreateBitCast(Ops[0], Int128Ty); in EmitAArch64BuiltinExpr()
10116 Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy); in EmitAArch64BuiltinExpr()
10125 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10126 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; in EmitAArch64BuiltinExpr()
10129 Ops[0] = Builder.CreateBitCast(Ops[0], InTy); in EmitAArch64BuiltinExpr()
10131 return Builder.CreateUIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
10132 return Builder.CreateSIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
10142 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10145 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64) in EmitAArch64BuiltinExpr()
10147 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32) in EmitAArch64BuiltinExpr()
10151 Ops[0] = Builder.CreateBitCast(Ops[0], InTy); in EmitAArch64BuiltinExpr()
10153 return Builder.CreateUIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
10154 return Builder.CreateSIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
10170 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10194 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvt"); in EmitAArch64BuiltinExpr()
10195 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
10205 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10213 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]); break; in EmitAArch64BuiltinExpr()
10215 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]); break; in EmitAArch64BuiltinExpr()
10217 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "facg"); in EmitAArch64BuiltinExpr()
10218 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
10226 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10234 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n"); in EmitAArch64BuiltinExpr()
10235 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
10243 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10248 Ops[0] = Builder.CreateSExt(Ops[0], InTy, "sext"); in EmitAArch64BuiltinExpr()
10252 Ops[0] = Builder.CreateZExt(Ops[0], InTy); in EmitAArch64BuiltinExpr()
10255 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n"); in EmitAArch64BuiltinExpr()
10297 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10299 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
10305 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10307 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
10313 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10315 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
10321 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10323 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
10329 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10331 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
10335 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10336 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
10337 Ops[0] = in EmitAArch64BuiltinExpr()
10338 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty)); in EmitAArch64BuiltinExpr()
10339 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqzd"); in EmitAArch64BuiltinExpr()
10355 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10356 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
10357 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); in EmitAArch64BuiltinExpr()
10359 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10361 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10362 return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd"); in EmitAArch64BuiltinExpr()
10378 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10379 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy); in EmitAArch64BuiltinExpr()
10380 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy); in EmitAArch64BuiltinExpr()
10382 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10384 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10385 return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd"); in EmitAArch64BuiltinExpr()
10401 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10402 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
10403 Ops[1] = Builder.CreateBitCast(Ops[1], HalfTy); in EmitAArch64BuiltinExpr()
10405 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10407 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10408 return Builder.CreateSExt(Ops[0], Int16Ty, "vcmpd"); in EmitAArch64BuiltinExpr()
10434 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10435 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
10436 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
10437 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10438 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd"); in EmitAArch64BuiltinExpr()
10442 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10443 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
10444 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
10445 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10446 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], in EmitAArch64BuiltinExpr()
10448 return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd"); in EmitAArch64BuiltinExpr()
10462 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
10463 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
10466 Ops[1] = in EmitAArch64BuiltinExpr()
10467 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(DoubleTy, 1)); in EmitAArch64BuiltinExpr()
10468 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
10469 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
10472 Ops[1] = in EmitAArch64BuiltinExpr()
10473 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(DoubleTy, 2)); in EmitAArch64BuiltinExpr()
10474 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
10475 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
10479 Ops[0] = in EmitAArch64BuiltinExpr()
10480 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int8Ty, 8)); in EmitAArch64BuiltinExpr()
10481 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10485 Ops[0] = in EmitAArch64BuiltinExpr()
10486 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int8Ty, 16)); in EmitAArch64BuiltinExpr()
10487 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10491 Ops[0] = in EmitAArch64BuiltinExpr()
10492 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int16Ty, 4)); in EmitAArch64BuiltinExpr()
10493 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10497 Ops[0] = in EmitAArch64BuiltinExpr()
10498 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int16Ty, 8)); in EmitAArch64BuiltinExpr()
10499 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10503 Ops[0] = in EmitAArch64BuiltinExpr()
10504 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int32Ty, 2)); in EmitAArch64BuiltinExpr()
10505 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10508 Ops[0] = in EmitAArch64BuiltinExpr()
10509 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 2)); in EmitAArch64BuiltinExpr()
10510 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10514 Ops[0] = in EmitAArch64BuiltinExpr()
10515 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int32Ty, 4)); in EmitAArch64BuiltinExpr()
10516 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10520 Ops[0] = in EmitAArch64BuiltinExpr()
10521 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 1)); in EmitAArch64BuiltinExpr()
10522 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10525 Ops[0] = in EmitAArch64BuiltinExpr()
10526 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 1)); in EmitAArch64BuiltinExpr()
10527 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10531 Ops[0] = in EmitAArch64BuiltinExpr()
10532 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 2)); in EmitAArch64BuiltinExpr()
10533 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10536 Ops[0] = in EmitAArch64BuiltinExpr()
10537 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 2)); in EmitAArch64BuiltinExpr()
10538 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10541 Ops[0] = in EmitAArch64BuiltinExpr()
10542 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 1)); in EmitAArch64BuiltinExpr()
10543 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10547 Ops[0] = in EmitAArch64BuiltinExpr()
10548 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 4)); in EmitAArch64BuiltinExpr()
10549 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10553 Ops[0] = in EmitAArch64BuiltinExpr()
10554 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 2)); in EmitAArch64BuiltinExpr()
10555 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10558 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10559 return Builder.CreateFAdd(Ops[0], Ops[1], "vaddh"); in EmitAArch64BuiltinExpr()
10561 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10562 return Builder.CreateFSub(Ops[0], Ops[1], "vsubh"); in EmitAArch64BuiltinExpr()
10564 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10565 return Builder.CreateFMul(Ops[0], Ops[1], "vmulh"); in EmitAArch64BuiltinExpr()
10567 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10568 return Builder.CreateFDiv(Ops[0], Ops[1], "vdivh"); in EmitAArch64BuiltinExpr()
10573 {EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2)), Ops[0]}); in EmitAArch64BuiltinExpr()
10582 {Sub, EmitScalarExpr(E->getArg(2)), Ops[0]}); in EmitAArch64BuiltinExpr()
10586 return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd"); in EmitAArch64BuiltinExpr()
10589 return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd"); in EmitAArch64BuiltinExpr()
10593 ProductOps.push_back(vectorWrapScalar16(Ops[1])); in EmitAArch64BuiltinExpr()
10596 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), in EmitAArch64BuiltinExpr()
10599 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); in EmitAArch64BuiltinExpr()
10604 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
10607 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10608 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
10610 Ops, "vqshlu_n"); in EmitAArch64BuiltinExpr()
10617 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10618 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
10619 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n"); in EmitAArch64BuiltinExpr()
10626 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10627 int SV = cast<ConstantInt>(Ops[1])->getSExtValue(); in EmitAArch64BuiltinExpr()
10628 Ops[1] = ConstantInt::get(Int64Ty, -SV); in EmitAArch64BuiltinExpr()
10629 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n"); in EmitAArch64BuiltinExpr()
10636 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
10637 Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2)))); in EmitAArch64BuiltinExpr()
10638 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty), in EmitAArch64BuiltinExpr()
10639 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)}); in EmitAArch64BuiltinExpr()
10640 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty)); in EmitAArch64BuiltinExpr()
10646 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n"); in EmitAArch64BuiltinExpr()
10651 Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), in EmitAArch64BuiltinExpr()
10661 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt), in EmitAArch64BuiltinExpr()
10666 Ops[1] = Builder.CreateAShr( in EmitAArch64BuiltinExpr()
10667 Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), in EmitAArch64BuiltinExpr()
10670 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10678 return Ops[0]; in EmitAArch64BuiltinExpr()
10679 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt), in EmitAArch64BuiltinExpr()
10681 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10687 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), in EmitAArch64BuiltinExpr()
10690 ProductOps.push_back(vectorWrapScalar16(Ops[1])); in EmitAArch64BuiltinExpr()
10691 ProductOps.push_back(vectorWrapScalar16(Ops[2])); in EmitAArch64BuiltinExpr()
10693 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), in EmitAArch64BuiltinExpr()
10696 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); in EmitAArch64BuiltinExpr()
10697 Ops.pop_back(); in EmitAArch64BuiltinExpr()
10703 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
10708 ProductOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
10710 Ops[1] = in EmitAArch64BuiltinExpr()
10717 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
10723 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), in EmitAArch64BuiltinExpr()
10726 ProductOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
10727 ProductOps.push_back(Ops[2]); in EmitAArch64BuiltinExpr()
10728 Ops[1] = in EmitAArch64BuiltinExpr()
10731 Ops.pop_back(); in EmitAArch64BuiltinExpr()
10737 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
10742 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10748 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10775 Builtin->NameHint, Builtin->TypeModifier, E, Ops, in EmitAArch64BuiltinExpr()
10778 if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops, Arch)) in EmitAArch64BuiltinExpr()
10787 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
10788 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
10789 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
10791 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl"); in EmitAArch64BuiltinExpr()
10792 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl"); in EmitAArch64BuiltinExpr()
10793 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl"); in EmitAArch64BuiltinExpr()
10794 return Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
10800 Value *Addend = Ops[0]; in EmitAArch64BuiltinExpr()
10801 Value *Multiplicand = Ops[1]; in EmitAArch64BuiltinExpr()
10802 Value *LaneSource = Ops[2]; in EmitAArch64BuiltinExpr()
10803 Ops[0] = Multiplicand; in EmitAArch64BuiltinExpr()
10804 Ops[1] = LaneSource; in EmitAArch64BuiltinExpr()
10805 Ops[2] = Addend; in EmitAArch64BuiltinExpr()
10812 llvm::Constant *cst = cast<Constant>(Ops[3]); in EmitAArch64BuiltinExpr()
10814 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy); in EmitAArch64BuiltinExpr()
10815 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane"); in EmitAArch64BuiltinExpr()
10817 Ops.pop_back(); in EmitAArch64BuiltinExpr()
10820 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla"); in EmitAArch64BuiltinExpr()
10826 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
10827 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); in EmitAArch64BuiltinExpr()
10830 Ops[2] = Builder.CreateBitCast(Ops[2], VTy); in EmitAArch64BuiltinExpr()
10831 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); in EmitAArch64BuiltinExpr()
10835 DoubleTy, {Ops[1], Ops[2], Ops[0]}); in EmitAArch64BuiltinExpr()
10838 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
10839 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
10843 Ops[2] = Builder.CreateBitCast(Ops[2], STy); in EmitAArch64BuiltinExpr()
10845 cast<ConstantInt>(Ops[3])); in EmitAArch64BuiltinExpr()
10846 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane"); in EmitAArch64BuiltinExpr()
10850 {Ops[2], Ops[1], Ops[0]}); in EmitAArch64BuiltinExpr()
10853 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
10854 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
10856 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
10857 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3])); in EmitAArch64BuiltinExpr()
10860 {Ops[2], Ops[1], Ops[0]}); in EmitAArch64BuiltinExpr()
10868 Ops.push_back(EmitScalarExpr(E->getArg(3))); in EmitAArch64BuiltinExpr()
10870 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); in EmitAArch64BuiltinExpr()
10873 {Ops[1], Ops[2], Ops[0]}); in EmitAArch64BuiltinExpr()
10879 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); in EmitAArch64BuiltinExpr()
10885 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax"); in EmitAArch64BuiltinExpr()
10887 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10889 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmax"); in EmitAArch64BuiltinExpr()
10896 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin"); in EmitAArch64BuiltinExpr()
10898 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10900 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmin"); in EmitAArch64BuiltinExpr()
10907 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd"); in EmitAArch64BuiltinExpr()
10918 TmpOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
10921 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType()); in EmitAArch64BuiltinExpr()
10929 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin"); in EmitAArch64BuiltinExpr()
10935 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax"); in EmitAArch64BuiltinExpr()
10939 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm"); in EmitAArch64BuiltinExpr()
10941 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10943 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vminnm"); in EmitAArch64BuiltinExpr()
10947 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm"); in EmitAArch64BuiltinExpr()
10949 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10951 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmaxnm"); in EmitAArch64BuiltinExpr()
10953 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10955 Ops, "vrecps"); in EmitAArch64BuiltinExpr()
10958 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10960 Ops, "vrecps"); in EmitAArch64BuiltinExpr()
10962 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10964 Ops, "vrecps"); in EmitAArch64BuiltinExpr()
10967 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n"); in EmitAArch64BuiltinExpr()
10970 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n"); in EmitAArch64BuiltinExpr()
10973 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n"); in EmitAArch64BuiltinExpr()
10976 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n"); in EmitAArch64BuiltinExpr()
10979 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n"); in EmitAArch64BuiltinExpr()
10981 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10985 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrnda"); in EmitAArch64BuiltinExpr()
10992 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda"); in EmitAArch64BuiltinExpr()
10995 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10999 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndi"); in EmitAArch64BuiltinExpr()
11002 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11006 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndm"); in EmitAArch64BuiltinExpr()
11013 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm"); in EmitAArch64BuiltinExpr()
11016 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11020 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndn"); in EmitAArch64BuiltinExpr()
11027 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn"); in EmitAArch64BuiltinExpr()
11030 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11034 return EmitNeonCall(CGM.getIntrinsic(Int, FloatTy), Ops, "vrndn"); in EmitAArch64BuiltinExpr()
11037 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11041 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndp"); in EmitAArch64BuiltinExpr()
11048 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp"); in EmitAArch64BuiltinExpr()
11051 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11055 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndx"); in EmitAArch64BuiltinExpr()
11062 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx"); in EmitAArch64BuiltinExpr()
11065 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11069 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndz"); in EmitAArch64BuiltinExpr()
11073 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11075 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd32x"); in EmitAArch64BuiltinExpr()
11079 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11081 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd32z"); in EmitAArch64BuiltinExpr()
11085 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11087 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd64x"); in EmitAArch64BuiltinExpr()
11091 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11093 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd64z"); in EmitAArch64BuiltinExpr()
11100 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz"); in EmitAArch64BuiltinExpr()
11104 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
11106 return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") in EmitAArch64BuiltinExpr()
11107 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
11112 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); in EmitAArch64BuiltinExpr()
11114 return Builder.CreateFPExt(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
11120 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); in EmitAArch64BuiltinExpr()
11122 return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
11139 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtz"); in EmitAArch64BuiltinExpr()
11155 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta"); in EmitAArch64BuiltinExpr()
11171 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm"); in EmitAArch64BuiltinExpr()
11187 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn"); in EmitAArch64BuiltinExpr()
11203 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp"); in EmitAArch64BuiltinExpr()
11208 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx"); in EmitAArch64BuiltinExpr()
11214 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
11215 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract"); in EmitAArch64BuiltinExpr()
11216 Ops.pop_back(); in EmitAArch64BuiltinExpr()
11218 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmulx"); in EmitAArch64BuiltinExpr()
11226 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
11229 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); in EmitAArch64BuiltinExpr()
11230 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract"); in EmitAArch64BuiltinExpr()
11231 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11241 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm"); in EmitAArch64BuiltinExpr()
11246 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm"); in EmitAArch64BuiltinExpr()
11249 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11253 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vsqrt"); in EmitAArch64BuiltinExpr()
11260 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
11261 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt"); in EmitAArch64BuiltinExpr()
11266 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit"); in EmitAArch64BuiltinExpr()
11277 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11278 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
11279 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
11289 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11290 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
11291 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11301 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11302 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
11303 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
11313 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11314 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
11315 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11322 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11323 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
11324 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
11331 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11332 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
11333 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11340 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11341 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
11342 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
11349 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11350 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
11351 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11358 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11359 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
11360 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
11367 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11368 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
11369 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11376 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11377 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
11378 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
11385 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11386 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
11387 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11394 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11395 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
11396 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
11403 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11404 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
11405 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
11412 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11413 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
11414 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
11421 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11422 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
11423 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11430 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11431 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
11432 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
11439 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11440 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
11441 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11448 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11449 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
11450 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
11457 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11458 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
11459 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11466 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11467 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
11468 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
11475 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11476 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
11477 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11484 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11485 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
11486 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
11493 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11494 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
11495 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
11502 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11503 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv"); in EmitAArch64BuiltinExpr()
11504 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
11511 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11512 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv"); in EmitAArch64BuiltinExpr()
11513 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
11520 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11521 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv"); in EmitAArch64BuiltinExpr()
11522 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
11529 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11530 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv"); in EmitAArch64BuiltinExpr()
11531 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
11534 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
11536 return Builder.CreateFMul(Ops[0], RHS); in EmitAArch64BuiltinExpr()
11543 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11544 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
11545 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11552 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11553 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
11560 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11561 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
11562 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11569 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11570 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
11577 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11578 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
11579 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11586 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11587 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
11594 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11595 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
11596 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11603 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11604 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
11610 return EmitNeonCall(Intrin, Ops, "vsri_n"); in EmitAArch64BuiltinExpr()
11616 return EmitNeonCall(Intrin, Ops, "vsli_n"); in EmitAArch64BuiltinExpr()
11620 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
11621 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); in EmitAArch64BuiltinExpr()
11622 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11627 TmpOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
11628 TmpOps.push_back(Ops[2]); in EmitAArch64BuiltinExpr()
11631 Ops[0] = Builder.CreateBitCast(Ops[0], VTy); in EmitAArch64BuiltinExpr()
11632 return Builder.CreateAdd(Ops[0], tmp); in EmitAArch64BuiltinExpr()
11636 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); in EmitAArch64BuiltinExpr()
11637 return Builder.CreateAlignedLoad(VTy, Ops[0], PtrOp0.getAlignment()); in EmitAArch64BuiltinExpr()
11641 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); in EmitAArch64BuiltinExpr()
11642 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); in EmitAArch64BuiltinExpr()
11643 return Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.getAlignment()); in EmitAArch64BuiltinExpr()
11646 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
11648 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
11649 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], in EmitAArch64BuiltinExpr()
11651 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane"); in EmitAArch64BuiltinExpr()
11657 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
11658 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], in EmitAArch64BuiltinExpr()
11661 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI); in EmitAArch64BuiltinExpr()
11662 return EmitNeonSplat(Ops[0], CI); in EmitAArch64BuiltinExpr()
11666 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
11667 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); in EmitAArch64BuiltinExpr()
11668 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
11669 return Builder.CreateAlignedStore(Ops[1], Builder.CreateBitCast(Ops[0], Ty), in EmitAArch64BuiltinExpr()
11674 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
11677 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); in EmitAArch64BuiltinExpr()
11678 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
11679 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
11680 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
11685 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
11688 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); in EmitAArch64BuiltinExpr()
11689 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
11690 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
11691 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
11696 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
11699 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); in EmitAArch64BuiltinExpr()
11700 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
11701 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
11702 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
11708 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
11711 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); in EmitAArch64BuiltinExpr()
11712 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
11713 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
11714 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
11720 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
11723 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); in EmitAArch64BuiltinExpr()
11724 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
11725 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
11726 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
11732 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
11735 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); in EmitAArch64BuiltinExpr()
11736 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
11737 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
11738 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
11742 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
11744 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end()); in EmitAArch64BuiltinExpr()
11745 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
11746 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
11747 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); in EmitAArch64BuiltinExpr()
11748 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane"); in EmitAArch64BuiltinExpr()
11749 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
11750 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
11751 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
11755 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
11757 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end()); in EmitAArch64BuiltinExpr()
11758 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
11759 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
11760 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); in EmitAArch64BuiltinExpr()
11761 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); in EmitAArch64BuiltinExpr()
11762 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane"); in EmitAArch64BuiltinExpr()
11763 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
11764 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
11765 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
11769 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
11771 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end()); in EmitAArch64BuiltinExpr()
11772 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
11773 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
11774 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); in EmitAArch64BuiltinExpr()
11775 Ops[4] = Builder.CreateBitCast(Ops[4], Ty); in EmitAArch64BuiltinExpr()
11776 Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty); in EmitAArch64BuiltinExpr()
11777 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld4_lane"); in EmitAArch64BuiltinExpr()
11778 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
11779 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
11780 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
11784 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
11785 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() }; in EmitAArch64BuiltinExpr()
11787 Ops, ""); in EmitAArch64BuiltinExpr()
11791 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
11792 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty); in EmitAArch64BuiltinExpr()
11793 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; in EmitAArch64BuiltinExpr()
11795 Ops, ""); in EmitAArch64BuiltinExpr()
11799 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
11800 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; in EmitAArch64BuiltinExpr()
11802 Ops, ""); in EmitAArch64BuiltinExpr()
11806 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
11807 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); in EmitAArch64BuiltinExpr()
11808 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; in EmitAArch64BuiltinExpr()
11810 Ops, ""); in EmitAArch64BuiltinExpr()
11814 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
11815 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; in EmitAArch64BuiltinExpr()
11817 Ops, ""); in EmitAArch64BuiltinExpr()
11821 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
11822 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); in EmitAArch64BuiltinExpr()
11823 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() }; in EmitAArch64BuiltinExpr()
11825 Ops, ""); in EmitAArch64BuiltinExpr()
11829 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitAArch64BuiltinExpr()
11830 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
11831 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
11840 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
11841 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn"); in EmitAArch64BuiltinExpr()
11848 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitAArch64BuiltinExpr()
11849 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
11850 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
11858 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
11859 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp"); in EmitAArch64BuiltinExpr()
11866 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitAArch64BuiltinExpr()
11867 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
11868 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
11877 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
11878 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip"); in EmitAArch64BuiltinExpr()
11885 Ops, "vtbl1"); in EmitAArch64BuiltinExpr()
11889 Ops, "vtbl2"); in EmitAArch64BuiltinExpr()
11893 Ops, "vtbl3"); in EmitAArch64BuiltinExpr()
11897 Ops, "vtbl4"); in EmitAArch64BuiltinExpr()
11901 Ops, "vtbx1"); in EmitAArch64BuiltinExpr()
11905 Ops, "vtbx2"); in EmitAArch64BuiltinExpr()
11909 Ops, "vtbx3"); in EmitAArch64BuiltinExpr()
11913 Ops, "vtbx4"); in EmitAArch64BuiltinExpr()
11918 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd"); in EmitAArch64BuiltinExpr()
11923 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd"); in EmitAArch64BuiltinExpr()
12037 BuildVector(ArrayRef<llvm::Value*> Ops) { in BuildVector() argument
12038 assert((Ops.size() & (Ops.size() - 1)) == 0 && in BuildVector()
12041 for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i) in BuildVector()
12042 AllConstants &= isa<Constant>(Ops[i]); in BuildVector()
12047 for (unsigned i = 0, e = Ops.size(); i != e; ++i) in BuildVector()
12048 CstOps.push_back(cast<Constant>(Ops[i])); in BuildVector()
12054 llvm::FixedVectorType::get(Ops[0]->getType(), Ops.size())); in BuildVector()
12056 for (unsigned i = 0, e = Ops.size(); i != e; ++i) in BuildVector()
12057 Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i)); in BuildVector()
12084 static Value *EmitX86MaskedStore(CodeGenFunction &CGF, ArrayRef<Value *> Ops, in EmitX86MaskedStore() argument
12087 Value *Ptr = CGF.Builder.CreateBitCast(Ops[0], in EmitX86MaskedStore()
12088 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitX86MaskedStore()
12091 CGF, Ops[2], in EmitX86MaskedStore()
12092 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements()); in EmitX86MaskedStore()
12094 return CGF.Builder.CreateMaskedStore(Ops[1], Ptr, Alignment, MaskVec); in EmitX86MaskedStore()
12097 static Value *EmitX86MaskedLoad(CodeGenFunction &CGF, ArrayRef<Value *> Ops, in EmitX86MaskedLoad() argument
12100 llvm::Type *Ty = Ops[1]->getType(); in EmitX86MaskedLoad()
12102 CGF.Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitX86MaskedLoad()
12105 CGF, Ops[2], cast<llvm::FixedVectorType>(Ty)->getNumElements()); in EmitX86MaskedLoad()
12107 return CGF.Builder.CreateMaskedLoad(Ty, Ptr, Alignment, MaskVec, Ops[1]); in EmitX86MaskedLoad()
12111 ArrayRef<Value *> Ops) { in EmitX86ExpandLoad() argument
12112 auto *ResultTy = cast<llvm::VectorType>(Ops[1]->getType()); in EmitX86ExpandLoad()
12116 Value *Ptr = CGF.Builder.CreateBitCast(Ops[0], in EmitX86ExpandLoad()
12120 CGF, Ops[2], cast<FixedVectorType>(ResultTy)->getNumElements()); in EmitX86ExpandLoad()
12124 return CGF.Builder.CreateCall(F, { Ptr, MaskVec, Ops[1] }); in EmitX86ExpandLoad()
12128 ArrayRef<Value *> Ops, in EmitX86CompressExpand() argument
12130 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType()); in EmitX86CompressExpand()
12132 Value *MaskVec = getMaskVecValue(CGF, Ops[2], ResultTy->getNumElements()); in EmitX86CompressExpand()
12137 return CGF.Builder.CreateCall(F, { Ops[0], Ops[1], MaskVec }); in EmitX86CompressExpand()
12141 ArrayRef<Value *> Ops) { in EmitX86CompressStore() argument
12142 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType()); in EmitX86CompressStore()
12146 Value *Ptr = CGF.Builder.CreateBitCast(Ops[0], in EmitX86CompressStore()
12149 Value *MaskVec = getMaskVecValue(CGF, Ops[2], ResultTy->getNumElements()); in EmitX86CompressStore()
12153 return CGF.Builder.CreateCall(F, { Ops[1], Ptr, MaskVec }); in EmitX86CompressStore()
12157 ArrayRef<Value *> Ops, in EmitX86MaskLogic() argument
12159 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86MaskLogic()
12160 Value *LHS = getMaskVecValue(CGF, Ops[0], NumElts); in EmitX86MaskLogic()
12161 Value *RHS = getMaskVecValue(CGF, Ops[1], NumElts); in EmitX86MaskLogic()
12167 Ops[0]->getType()); in EmitX86MaskLogic()
12188 static Value *EmitX86vpcom(CodeGenFunction &CGF, ArrayRef<Value *> Ops, in EmitX86vpcom() argument
12190 Value *Op0 = Ops[0]; in EmitX86vpcom()
12191 Value *Op1 = Ops[1]; in EmitX86vpcom()
12193 uint64_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7; in EmitX86vpcom()
12280 bool Signed, ArrayRef<Value *> Ops) { in EmitX86MaskedCompare() argument
12281 assert((Ops.size() == 2 || Ops.size() == 4) && in EmitX86MaskedCompare()
12284 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86MaskedCompare()
12304 Cmp = CGF.Builder.CreateICmp(Pred, Ops[0], Ops[1]); in EmitX86MaskedCompare()
12308 if (Ops.size() == 4) in EmitX86MaskedCompare()
12309 MaskIn = Ops[3]; in EmitX86MaskedCompare()
12320 ArrayRef<Value *> Ops, bool IsSigned) { in EmitX86ConvertIntToFp() argument
12321 unsigned Rnd = cast<llvm::ConstantInt>(Ops[3])->getZExtValue(); in EmitX86ConvertIntToFp()
12322 llvm::Type *Ty = Ops[1]->getType(); in EmitX86ConvertIntToFp()
12328 Function *F = CGF.CGM.getIntrinsic(IID, { Ty, Ops[0]->getType() }); in EmitX86ConvertIntToFp()
12329 Res = CGF.Builder.CreateCall(F, { Ops[0], Ops[3] }); in EmitX86ConvertIntToFp()
12332 Res = IsSigned ? CGF.Builder.CreateSIToFP(Ops[0], Ty) in EmitX86ConvertIntToFp()
12333 : CGF.Builder.CreateUIToFP(Ops[0], Ty); in EmitX86ConvertIntToFp()
12336 return EmitX86Select(CGF, Ops[2], Res, Ops[1]); in EmitX86ConvertIntToFp()
12341 ArrayRef<Value *> Ops, unsigned BuiltinID, in EmitX86FMAExpr() argument
12396 Value *A = Ops[0]; in EmitX86FMAExpr()
12397 Value *B = Ops[1]; in EmitX86FMAExpr()
12398 Value *C = Ops[2]; in EmitX86FMAExpr()
12407 (cast<llvm::ConstantInt>(Ops.back())->getZExtValue() != (uint64_t)4 || in EmitX86FMAExpr()
12410 Res = CGF.Builder.CreateCall(Intr, {A, B, C, Ops.back() }); in EmitX86FMAExpr()
12433 MaskFalseVal = Ops[0]; in EmitX86FMAExpr()
12441 MaskFalseVal = Constant::getNullValue(Ops[0]->getType()); in EmitX86FMAExpr()
12455 MaskFalseVal = Ops[2]; in EmitX86FMAExpr()
12460 return EmitX86Select(CGF, Ops[3], Res, MaskFalseVal); in EmitX86FMAExpr()
12466 MutableArrayRef<Value *> Ops, Value *Upper, in EmitScalarFMAExpr() argument
12470 if (Ops.size() > 4) in EmitScalarFMAExpr()
12471 Rnd = cast<llvm::ConstantInt>(Ops[4])->getZExtValue(); in EmitScalarFMAExpr()
12474 Ops[2] = CGF.Builder.CreateFNeg(Ops[2]); in EmitScalarFMAExpr()
12476 Ops[0] = CGF.Builder.CreateExtractElement(Ops[0], (uint64_t)0); in EmitScalarFMAExpr()
12477 Ops[1] = CGF.Builder.CreateExtractElement(Ops[1], (uint64_t)0); in EmitScalarFMAExpr()
12478 Ops[2] = CGF.Builder.CreateExtractElement(Ops[2], (uint64_t)0); in EmitScalarFMAExpr()
12483 switch (Ops[0]->getType()->getPrimitiveSizeInBits()) { in EmitScalarFMAExpr()
12497 {Ops[0], Ops[1], Ops[2], Ops[4]}); in EmitScalarFMAExpr()
12501 Intrinsic::experimental_constrained_fma, Ops[0]->getType()); in EmitScalarFMAExpr()
12502 Res = CGF.Builder.CreateConstrainedFPCall(FMA, Ops.slice(0, 3)); in EmitScalarFMAExpr()
12504 Function *FMA = CGF.CGM.getIntrinsic(Intrinsic::fma, Ops[0]->getType()); in EmitScalarFMAExpr()
12505 Res = CGF.Builder.CreateCall(FMA, Ops.slice(0, 3)); in EmitScalarFMAExpr()
12508 if (Ops.size() > 3) { in EmitScalarFMAExpr()
12510 : Ops[PTIdx]; in EmitScalarFMAExpr()
12518 Res = EmitX86ScalarSelect(CGF, Ops[3], Res, PassThru); in EmitScalarFMAExpr()
12524 ArrayRef<Value *> Ops) { in EmitX86Muldq() argument
12525 llvm::Type *Ty = Ops[0]->getType(); in EmitX86Muldq()
12529 Value *LHS = CGF.Builder.CreateBitCast(Ops[0], Ty); in EmitX86Muldq()
12530 Value *RHS = CGF.Builder.CreateBitCast(Ops[1], Ty); in EmitX86Muldq()
12553 ArrayRef<Value *> Ops) { in EmitX86Ternlog() argument
12554 llvm::Type *Ty = Ops[0]->getType(); in EmitX86Ternlog()
12575 Ops.drop_back()); in EmitX86Ternlog()
12576 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) : Ops[0]; in EmitX86Ternlog()
12577 return EmitX86Select(CGF, Ops[4], Ternlog, PassThru); in EmitX86Ternlog()
12596 ArrayRef<Value *> Ops, in EmitX86CvtF16ToFloatExpr() argument
12598 assert((Ops.size() == 1 || Ops.size() == 3 || Ops.size() == 4) && in EmitX86CvtF16ToFloatExpr()
12602 if (Ops.size() == 4 && cast<llvm::ConstantInt>(Ops[3])->getZExtValue() != 4) { in EmitX86CvtF16ToFloatExpr()
12605 return CGF.Builder.CreateCall(F, {Ops[0], Ops[1], Ops[2], Ops[3]}); in EmitX86CvtF16ToFloatExpr()
12609 Value *Src = Ops[0]; in EmitX86CvtF16ToFloatExpr()
12626 if (Ops.size() >= 3) in EmitX86CvtF16ToFloatExpr()
12627 Res = EmitX86Select(CGF, Ops[2], Res, Ops[1]); in EmitX86CvtF16ToFloatExpr()
12634 ArrayRef<Value *> Ops) { in EmitX86CvtBF16ToFloatExpr() argument
12636 Value *ZeroExt = CGF.Builder.CreateZExt(Ops[0], Int32Ty); in EmitX86CvtBF16ToFloatExpr()
12777 SmallVector<Value*, 4> Ops; in EmitX86BuiltinExpr() local
12790 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitX86BuiltinExpr()
12796 Ops.push_back(llvm::ConstantInt::get( in EmitX86BuiltinExpr()
12806 auto getCmpIntrinsicCall = [this, &Ops](Intrinsic::ID ID, unsigned Imm) { in EmitX86BuiltinExpr()
12807 Ops.push_back(llvm::ConstantInt::get(Int8Ty, Imm)); in EmitX86BuiltinExpr()
12809 return Builder.CreateCall(F, Ops); in EmitX86BuiltinExpr()
12817 auto getVectorFCmpIR = [this, &Ops, E](CmpInst::Predicate Pred, in EmitX86BuiltinExpr()
12822 Cmp = Builder.CreateFCmpS(Pred, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
12824 Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
12825 llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
12834 Value *Address = Ops[0]; in EmitX86BuiltinExpr()
12835 ConstantInt *C = cast<ConstantInt>(Ops[1]); in EmitX86BuiltinExpr()
12844 Ops[0]); in EmitX86BuiltinExpr()
12864 Ops[0]); in EmitX86BuiltinExpr()
12870 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType()); in EmitX86BuiltinExpr()
12871 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)}); in EmitX86BuiltinExpr()
12876 Function *F = CGM.getIntrinsic(Intrinsic::cttz, Ops[0]->getType()); in EmitX86BuiltinExpr()
12877 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)}); in EmitX86BuiltinExpr()
12891 return Builder.CreateBitCast(BuildVector(Ops), in EmitX86BuiltinExpr()
12904 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
12905 uint64_t Index = cast<ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
12909 return Builder.CreateExtractElement(Ops[0], Index); in EmitX86BuiltinExpr()
12920 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
12921 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
12925 return Builder.CreateInsertElement(Ops[0], Ops[1], Index); in EmitX86BuiltinExpr()
12930 Builder.CreateStore(Ops[0], Tmp); in EmitX86BuiltinExpr()
12981 Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty); in EmitX86BuiltinExpr()
12982 Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty); in EmitX86BuiltinExpr()
12983 Ops[1] = Mhi; in EmitX86BuiltinExpr()
12984 Ops.push_back(Mlo); in EmitX86BuiltinExpr()
12985 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
12989 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_xgetbv), Ops); in EmitX86BuiltinExpr()
13008 return EmitX86MaskedStore(*this, Ops, Align(1)); in EmitX86BuiltinExpr()
13013 return EmitX86MaskedStore(*this, Ops, Align(1)); in EmitX86BuiltinExpr()
13029 return Builder.CreateCall(F, Ops); in EmitX86BuiltinExpr()
13043 return EmitX86SExtMask(*this, Ops[0], ConvertType(E->getType())); in EmitX86BuiltinExpr()
13057 return EmitX86ConvertToMask(*this, Ops[0]); in EmitX86BuiltinExpr()
13065 return EmitX86ConvertIntToFp(*this, E, Ops, /*IsSigned*/ true); in EmitX86BuiltinExpr()
13072 return EmitX86ConvertIntToFp(*this, E, Ops, /*IsSigned*/ false); in EmitX86BuiltinExpr()
13079 return EmitScalarFMAExpr(*this, E, Ops, Ops[0]); in EmitX86BuiltinExpr()
13082 return EmitScalarFMAExpr(*this, E, Ops, in EmitX86BuiltinExpr()
13083 Constant::getNullValue(Ops[0]->getType())); in EmitX86BuiltinExpr()
13087 return EmitScalarFMAExpr(*this, E, Ops, Ops[0], /*ZeroMask*/ true); in EmitX86BuiltinExpr()
13091 return EmitScalarFMAExpr(*this, E, Ops, Ops[2], /*ZeroMask*/ false, 2); in EmitX86BuiltinExpr()
13095 return EmitScalarFMAExpr(*this, E, Ops, Ops[2], /*ZeroMask*/ false, 2, in EmitX86BuiltinExpr()
13115 return EmitX86FMAExpr(*this, E, Ops, BuiltinID, /*IsAddSub*/ false); in EmitX86BuiltinExpr()
13128 return EmitX86FMAExpr(*this, E, Ops, BuiltinID, /*IsAddSub*/ true); in EmitX86BuiltinExpr()
13143 *this, Ops, in EmitX86BuiltinExpr()
13164 return EmitX86MaskedLoad(*this, Ops, Align(1)); in EmitX86BuiltinExpr()
13169 return EmitX86MaskedLoad(*this, Ops, Align(1)); in EmitX86BuiltinExpr()
13184 *this, Ops, in EmitX86BuiltinExpr()
13205 return EmitX86ExpandLoad(*this, Ops); in EmitX86BuiltinExpr()
13225 return EmitX86CompressStore(*this, Ops); in EmitX86BuiltinExpr()
13245 return EmitX86CompressExpand(*this, Ops, /*IsCompress*/false); in EmitX86BuiltinExpr()
13265 return EmitX86CompressExpand(*this, Ops, /*IsCompress*/true); in EmitX86BuiltinExpr()
13369 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(), in EmitX86BuiltinExpr()
13370 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements()); in EmitX86BuiltinExpr()
13371 Ops[3] = getMaskVecValue(*this, Ops[3], MinElts); in EmitX86BuiltinExpr()
13373 return Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
13478 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements(), in EmitX86BuiltinExpr()
13479 cast<llvm::FixedVectorType>(Ops[3]->getType())->getNumElements()); in EmitX86BuiltinExpr()
13480 Ops[1] = getMaskVecValue(*this, Ops[1], MinElts); in EmitX86BuiltinExpr()
13482 return Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
13504 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
13506 unsigned Index = cast<ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
13515 Value *Res = Builder.CreateShuffleVector(Ops[0], in EmitX86BuiltinExpr()
13519 if (Ops.size() == 4) in EmitX86BuiltinExpr()
13520 Res = EmitX86Select(*this, Ops[3], Res, Ops[2]); in EmitX86BuiltinExpr()
13541 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
13543 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements(); in EmitX86BuiltinExpr()
13545 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
13554 Value *Op1 = Builder.CreateShuffleVector(Ops[1], in EmitX86BuiltinExpr()
13565 return Builder.CreateShuffleVector(Ops[0], Op1, in EmitX86BuiltinExpr()
13571 Value *Res = Builder.CreateTrunc(Ops[0], Ops[1]->getType()); in EmitX86BuiltinExpr()
13572 return EmitX86Select(*this, Ops[2], Res, Ops[1]); in EmitX86BuiltinExpr()
13577 if (const auto *C = dyn_cast<Constant>(Ops[2])) in EmitX86BuiltinExpr()
13579 return Builder.CreateTrunc(Ops[0], Ops[1]->getType()); in EmitX86BuiltinExpr()
13596 return Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
13607 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
13608 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
13616 return Builder.CreateShuffleVector(Ops[0], Ops[1], in EmitX86BuiltinExpr()
13623 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
13624 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
13640 return Builder.CreateShuffleVector(Ops[0], makeArrayRef(Indices, NumElts), in EmitX86BuiltinExpr()
13646 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
13647 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
13663 return Builder.CreateShuffleVector(Ops[0], makeArrayRef(Indices, NumElts), in EmitX86BuiltinExpr()
13675 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
13676 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
13692 return Builder.CreateShuffleVector(Ops[0], makeArrayRef(Indices, NumElts), in EmitX86BuiltinExpr()
13701 uint32_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
13702 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
13721 return Builder.CreateShuffleVector(Ops[0], Ops[1], in EmitX86BuiltinExpr()
13729 unsigned Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
13730 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
13739 return Builder.CreateShuffleVector(Ops[0], makeArrayRef(Indices, NumElts), in EmitX86BuiltinExpr()
13745 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
13748 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
13760 Ops[1] = Ops[0]; in EmitX86BuiltinExpr()
13761 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
13775 return Builder.CreateShuffleVector(Ops[1], Ops[0], in EmitX86BuiltinExpr()
13786 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
13787 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
13796 return Builder.CreateShuffleVector(Ops[1], Ops[0], in EmitX86BuiltinExpr()
13808 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
13809 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
13825 return Builder.CreateShuffleVector(Ops[0], Ops[1], in EmitX86BuiltinExpr()
13834 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
13836 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
13848 OutOps[l] = llvm::ConstantAggregateZero::get(Ops[0]->getType()); in EmitX86BuiltinExpr()
13850 OutOps[l] = Ops[1]; in EmitX86BuiltinExpr()
13852 OutOps[l] = Ops[0]; in EmitX86BuiltinExpr()
13873 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
13874 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
13893 Value *Cast = Builder.CreateBitCast(Ops[0], VecTy, "cast"); in EmitX86BuiltinExpr()
13898 return Builder.CreateBitCast(SV, Ops[0]->getType(), "cast"); in EmitX86BuiltinExpr()
13903 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
13904 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
13923 Value *Cast = Builder.CreateBitCast(Ops[0], VecTy, "cast"); in EmitX86BuiltinExpr()
13934 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
13935 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
13938 return llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
13940 Value *In = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
13950 return Builder.CreateBitCast(SV, Ops[0]->getType()); in EmitX86BuiltinExpr()
13956 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
13957 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
13960 return llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
13962 Value *In = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
13972 return Builder.CreateBitCast(SV, Ops[0]->getType()); in EmitX86BuiltinExpr()
13981 Value *Ptr = Ops[0]; in EmitX86BuiltinExpr()
13982 Value *Src = Ops[1]; in EmitX86BuiltinExpr()
14020 return EmitX86FunnelShift(*this, Ops[0], Ops[0], Ops[1], false); in EmitX86BuiltinExpr()
14033 return EmitX86FunnelShift(*this, Ops[0], Ops[0], Ops[1], true); in EmitX86BuiltinExpr()
14055 return EmitX86Select(*this, Ops[0], Ops[1], Ops[2]); in EmitX86BuiltinExpr()
14059 Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0); in EmitX86BuiltinExpr()
14060 Value *B = Builder.CreateExtractElement(Ops[2], (uint64_t)0); in EmitX86BuiltinExpr()
14061 A = EmitX86ScalarSelect(*this, Ops[0], A, B); in EmitX86BuiltinExpr()
14062 return Builder.CreateInsertElement(Ops[1], A, (uint64_t)0); in EmitX86BuiltinExpr()
14076 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7; in EmitX86BuiltinExpr()
14077 return EmitX86MaskedCompare(*this, CC, true, Ops); in EmitX86BuiltinExpr()
14091 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7; in EmitX86BuiltinExpr()
14092 return EmitX86MaskedCompare(*this, CC, false, Ops); in EmitX86BuiltinExpr()
14098 return EmitX86vpcom(*this, Ops, true); in EmitX86BuiltinExpr()
14103 return EmitX86vpcom(*this, Ops, false); in EmitX86BuiltinExpr()
14109 Value *Or = EmitX86MaskLogic(*this, Instruction::Or, Ops); in EmitX86BuiltinExpr()
14110 Value *C = llvm::Constant::getAllOnesValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
14118 Value *Or = EmitX86MaskLogic(*this, Instruction::Or, Ops); in EmitX86BuiltinExpr()
14119 Value *C = llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
14161 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
14162 Value *LHS = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
14163 Value *RHS = getMaskVecValue(*this, Ops[1], NumElts); in EmitX86BuiltinExpr()
14189 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
14190 Value *LHS = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
14191 Value *RHS = getMaskVecValue(*this, Ops[1], NumElts); in EmitX86BuiltinExpr()
14194 return Builder.CreateBitCast(Res, Ops[0]->getType()); in EmitX86BuiltinExpr()
14200 return EmitX86MaskLogic(*this, Instruction::And, Ops); in EmitX86BuiltinExpr()
14205 return EmitX86MaskLogic(*this, Instruction::And, Ops, true); in EmitX86BuiltinExpr()
14210 return EmitX86MaskLogic(*this, Instruction::Or, Ops); in EmitX86BuiltinExpr()
14215 return EmitX86MaskLogic(*this, Instruction::Xor, Ops, true); in EmitX86BuiltinExpr()
14220 return EmitX86MaskLogic(*this, Instruction::Xor, Ops); in EmitX86BuiltinExpr()
14225 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
14226 Value *Res = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
14228 Ops[0]->getType()); in EmitX86BuiltinExpr()
14237 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
14238 Value *Res = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
14239 return Builder.CreateBitCast(Res, Ops[0]->getType()); in EmitX86BuiltinExpr()
14245 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
14246 Value *LHS = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
14247 Value *RHS = getMaskVecValue(*this, Ops[1], NumElts); in EmitX86BuiltinExpr()
14262 return Builder.CreateBitCast(Res, Ops[0]->getType()); in EmitX86BuiltinExpr()
14271 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType()); in EmitX86BuiltinExpr()
14272 return Builder.CreateCall(F, {Ops[0],Builder.getInt1(false)}); in EmitX86BuiltinExpr()
14276 Value *A = Builder.CreateExtractElement(Ops[0], (uint64_t)0); in EmitX86BuiltinExpr()
14287 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0); in EmitX86BuiltinExpr()
14292 unsigned CC = cast<llvm::ConstantInt>(Ops[4])->getZExtValue(); in EmitX86BuiltinExpr()
14311 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
14313 Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0); in EmitX86BuiltinExpr()
14324 Value *Src = Builder.CreateExtractElement(Ops[2], (uint64_t)0); in EmitX86BuiltinExpr()
14325 A = EmitX86ScalarSelect(*this, Ops[3], A, Src); in EmitX86BuiltinExpr()
14326 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0); in EmitX86BuiltinExpr()
14337 if (Ops.size() == 2) { in EmitX86BuiltinExpr()
14338 unsigned CC = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
14357 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
14363 Ops[0]->getType()); in EmitX86BuiltinExpr()
14364 return Builder.CreateConstrainedFPCall(F, Ops[0]); in EmitX86BuiltinExpr()
14366 Function *F = CGM.getIntrinsic(Intrinsic::sqrt, Ops[0]->getType()); in EmitX86BuiltinExpr()
14367 return Builder.CreateCall(F, Ops[0]); in EmitX86BuiltinExpr()
14374 return EmitX86Muldq(*this, /*IsSigned*/false, Ops); in EmitX86BuiltinExpr()
14379 return EmitX86Muldq(*this, /*IsSigned*/true, Ops); in EmitX86BuiltinExpr()
14387 return EmitX86Ternlog(*this, /*ZeroMask*/false, Ops); in EmitX86BuiltinExpr()
14395 return EmitX86Ternlog(*this, /*ZeroMask*/true, Ops); in EmitX86BuiltinExpr()
14406 return EmitX86FunnelShift(*this, Ops[0], Ops[1], Ops[2], false); in EmitX86BuiltinExpr()
14418 return EmitX86FunnelShift(*this, Ops[1], Ops[0], Ops[2], true); in EmitX86BuiltinExpr()
14429 return EmitX86FunnelShift(*this, Ops[0], Ops[1], Ops[2], false); in EmitX86BuiltinExpr()
14441 return EmitX86FunnelShift(*this, Ops[1], Ops[0], Ops[2], true); in EmitX86BuiltinExpr()
14450 CGM.getIntrinsic(Intrinsic::vector_reduce_fadd, Ops[1]->getType()); in EmitX86BuiltinExpr()
14452 return Builder.CreateCall(F, {Ops[0], Ops[1]}); in EmitX86BuiltinExpr()
14460 CGM.getIntrinsic(Intrinsic::vector_reduce_fmul, Ops[1]->getType()); in EmitX86BuiltinExpr()
14462 return Builder.CreateCall(F, {Ops[0], Ops[1]}); in EmitX86BuiltinExpr()
14470 CGM.getIntrinsic(Intrinsic::vector_reduce_fmax, Ops[0]->getType()); in EmitX86BuiltinExpr()
14472 return Builder.CreateCall(F, {Ops[0]}); in EmitX86BuiltinExpr()
14480 CGM.getIntrinsic(Intrinsic::vector_reduce_fmin, Ops[0]->getType()); in EmitX86BuiltinExpr()
14482 return Builder.CreateCall(F, {Ops[0]}); in EmitX86BuiltinExpr()
14489 Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast"); in EmitX86BuiltinExpr()
14491 return Builder.CreateCall(F, Ops, "pswapd"); in EmitX86BuiltinExpr()
14524 Ops[0]); in EmitX86BuiltinExpr()
14549 { Ops[0], Ops[1], Ops[2] }); in EmitX86BuiltinExpr()
14551 Ops[3]); in EmitX86BuiltinExpr()
14565 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
14566 Value *MaskIn = Ops[2]; in EmitX86BuiltinExpr()
14567 Ops.erase(&Ops[2]); in EmitX86BuiltinExpr()
14601 Value *Fpclass = Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
14612 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
14637 Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID), {Ops[0], Ops[1]}); in EmitX86BuiltinExpr()
14640 Builder.CreateDefaultAlignedStore(Result, Ops[2]); in EmitX86BuiltinExpr()
14644 return Builder.CreateDefaultAlignedStore(Result, Ops[3]); in EmitX86BuiltinExpr()
14664 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
14671 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
14672 Value *MaskIn = Ops[2]; in EmitX86BuiltinExpr()
14673 Ops.erase(&Ops[2]); in EmitX86BuiltinExpr()
14689 Value *Shufbit = Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
14740 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x1f; in EmitX86BuiltinExpr()
14820 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
14821 Ops[3] = getMaskVecValue(*this, Ops[3], NumElts); in EmitX86BuiltinExpr()
14822 Value *Cmp = Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
14826 return Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
14837 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
14840 Cmp = Builder.CreateFCmpS(Pred, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
14842 Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
14843 return EmitX86MaskedCompareResult(*this, Cmp, NumElts, Ops[3]); in EmitX86BuiltinExpr()
14890 return EmitX86CvtF16ToFloatExpr(*this, Ops, ConvertType(E->getType())); in EmitX86BuiltinExpr()
14895 Ops[2] = getMaskVecValue( in EmitX86BuiltinExpr()
14896 *this, Ops[2], in EmitX86BuiltinExpr()
14897 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements()); in EmitX86BuiltinExpr()
14899 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
14902 return EmitX86CvtBF16ToFloatExpr(*this, E, Ops); in EmitX86BuiltinExpr()
14916 Value *Res = Builder.CreateCall(CGM.getIntrinsic(IID), Ops[0]); in EmitX86BuiltinExpr()
14917 return EmitX86Select(*this, Ops[2], Res, Ops[1]); in EmitX86BuiltinExpr()
14964 Value *LHS = Builder.CreateIntCast(Ops[0], Int64Ty, isSigned); in EmitX86BuiltinExpr()
14965 Value *RHS = Builder.CreateIntCast(Ops[1], Int64Ty, isSigned); in EmitX86BuiltinExpr()
14976 Value *LHS = Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned); in EmitX86BuiltinExpr()
14977 Value *RHS = Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned); in EmitX86BuiltinExpr()
15009 std::swap(Ops[0], Ops[1]); in EmitX86BuiltinExpr()
15010 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty); in EmitX86BuiltinExpr()
15011 return Builder.CreateCall(F, Ops); in EmitX86BuiltinExpr()
15028 return Builder.CreateMemSet(Ops[0], Ops[1], Ops[2], Align(1), true); in EmitX86BuiltinExpr()
15051 Builder.CreateIntToPtr(Ops[0], llvm::PointerType::get(IntTy, 257)); in EmitX86BuiltinExpr()
15063 Builder.CreateIntToPtr(Ops[0], llvm::PointerType::get(IntTy, 256)); in EmitX86BuiltinExpr()
15072 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), {Ops[0], Ops[1]}); in EmitX86BuiltinExpr()
15076 Value *Ptr = Builder.CreateConstGEP1_32(Int8Ty, Ops[2], i * 16); in EmitX86BuiltinExpr()
15088 Builder.CreateCall(CGM.getIntrinsic(IID), {Ops[0], Ops[1], Ops[2]}); in EmitX86BuiltinExpr()
15092 Value *Ptr = Builder.CreateConstGEP1_32(Int8Ty, Ops[3], i * 16); in EmitX86BuiltinExpr()
15127 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), {Ops[1], Ops[2]}); in EmitX86BuiltinExpr()
15140 Builder.CreateDefaultAlignedStore(Out, Ops[0]); in EmitX86BuiltinExpr()
15145 Builder.CreateDefaultAlignedStore(Zero, Ops[0]); in EmitX86BuiltinExpr()
15178 InOps[0] = Ops[2]; in EmitX86BuiltinExpr()
15180 Value *Ptr = Builder.CreateConstGEP1_32(Ty, Ops[1], i); in EmitX86BuiltinExpr()
15198 Value *Ptr = Builder.CreateConstGEP1_32(Extract->getType(), Ops[0], i); in EmitX86BuiltinExpr()
15207 Value *Ptr = Builder.CreateConstGEP1_32(Out->getType(), Ops[0], i); in EmitX86BuiltinExpr()
15222 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
15223 return EmitX86Select(*this, Ops[3], Call, Ops[0]); in EmitX86BuiltinExpr()
15231 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
15232 Value *And = Builder.CreateAnd(Ops[3], llvm::ConstantInt::get(Int8Ty, 1)); in EmitX86BuiltinExpr()
15233 return EmitX86Select(*this, And, Call, Ops[0]); in EmitX86BuiltinExpr()
15241 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
15243 return Builder.CreateShuffleVector(Call, Ops[2], Mask); in EmitX86BuiltinExpr()
15287 SmallVector<Value *, 2> Ops; in EmitPPCBuiltinExpr() local
15288 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitPPCBuiltinExpr()
15289 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitPPCBuiltinExpr()
15292 Ops[0] = Builder.CreateBitCast(Ops[0], Int8PtrTy); in EmitPPCBuiltinExpr()
15294 Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); in EmitPPCBuiltinExpr()
15295 Ops[0] = Builder.CreateGEP(Int8Ty, Ops[1], Ops[0]); in EmitPPCBuiltinExpr()
15296 Ops.pop_back(); in EmitPPCBuiltinExpr()
15342 return Builder.CreateCall(F, Ops, ""); in EmitPPCBuiltinExpr()
15358 SmallVector<Value *, 3> Ops; in EmitPPCBuiltinExpr() local
15359 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitPPCBuiltinExpr()
15360 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitPPCBuiltinExpr()
15361 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitPPCBuiltinExpr()
15364 Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); in EmitPPCBuiltinExpr()
15366 Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy); in EmitPPCBuiltinExpr()
15367 Ops[1] = Builder.CreateGEP(Int8Ty, Ops[2], Ops[1]); in EmitPPCBuiltinExpr()
15368 Ops.pop_back(); in EmitPPCBuiltinExpr()
15408 return Builder.CreateCall(F, Ops, ""); in EmitPPCBuiltinExpr()
15668 SmallVector<Value *, 2> Ops; in EmitPPCBuiltinExpr() local
15673 Ops.push_back(Builder.CreateBitCast(Op0, V1I128Ty)); in EmitPPCBuiltinExpr()
15674 Ops.push_back(Builder.CreateBitCast(Op1, V1I128Ty)); in EmitPPCBuiltinExpr()
15678 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops, ""); in EmitPPCBuiltinExpr()
15684 SmallVector<Value *, 3> Ops; in EmitPPCBuiltinExpr() local
15690 Ops.push_back(Builder.CreateBitCast(Op0, V1I128Ty)); in EmitPPCBuiltinExpr()
15691 Ops.push_back(Builder.CreateBitCast(Op1, V1I128Ty)); in EmitPPCBuiltinExpr()
15692 Ops.push_back(Builder.CreateBitCast(Op2, V1I128Ty)); in EmitPPCBuiltinExpr()
15709 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops, ""); in EmitPPCBuiltinExpr()
16145 SmallVector<Value *, 4> Ops; in EmitPPCBuiltinExpr() local
16148 Ops.push_back(EmitArrayToPointerDecay(E->getArg(i)).getPointer()); in EmitPPCBuiltinExpr()
16150 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitPPCBuiltinExpr()
16169 Value *Ptr = Builder.CreateBitCast(Ops[0], VTy->getPointerTo()); in EmitPPCBuiltinExpr()
16187 std::reverse(Ops.begin() + 1, Ops.end()); in EmitPPCBuiltinExpr()
16204 Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); in EmitPPCBuiltinExpr()
16205 Ops[0] = Builder.CreateGEP(Int8Ty, Ops[1], Ops[0]); in EmitPPCBuiltinExpr()
16207 Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy); in EmitPPCBuiltinExpr()
16208 Ops[1] = Builder.CreateGEP(Int8Ty, Ops[2], Ops[1]); in EmitPPCBuiltinExpr()
16210 Ops.pop_back(); in EmitPPCBuiltinExpr()
16212 return Builder.CreateCall(F, Ops, ""); in EmitPPCBuiltinExpr()
16220 for (unsigned i=1; i<Ops.size(); i++) in EmitPPCBuiltinExpr()
16221 CallOps.push_back(Ops[i]); in EmitPPCBuiltinExpr()
16224 return Builder.CreateAlignedStore(Call, Ops[0], MaybeAlign(64)); in EmitPPCBuiltinExpr()
18713 Value *Ops[18]; in EmitWebAssemblyBuiltinExpr() local
18715 Ops[OpIdx++] = EmitScalarExpr(E->getArg(0)); in EmitWebAssemblyBuiltinExpr()
18716 Ops[OpIdx++] = EmitScalarExpr(E->getArg(1)); in EmitWebAssemblyBuiltinExpr()
18721 Ops[OpIdx++] = llvm::ConstantInt::get(getLLVMContext(), *LaneConst); in EmitWebAssemblyBuiltinExpr()
18724 return Builder.CreateCall(Callee, Ops); in EmitWebAssemblyBuiltinExpr()
18915 SmallVector<llvm::Value*,5> Ops = { Base }; in EmitHexagonBuiltinExpr() local
18917 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitHexagonBuiltinExpr()
18919 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(IntID), Ops); in EmitHexagonBuiltinExpr()
19016 SmallVector<llvm::Value*,4> Ops; in EmitHexagonBuiltinExpr() local
19022 Ops.push_back(V2Q(EmitScalarExpr(PredOp))); in EmitHexagonBuiltinExpr()
19025 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitHexagonBuiltinExpr()
19026 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitHexagonBuiltinExpr()
19073 SmallVector<Value *, 4> Ops; in EmitRISCVBuiltinExpr() local
19095 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitRISCVBuiltinExpr()
19101 Ops.push_back(llvm::ConstantInt::get( in EmitRISCVBuiltinExpr()
19166 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType()); in EmitRISCVBuiltinExpr()
19167 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)}); in EmitRISCVBuiltinExpr()
19171 Function *F = CGM.getIntrinsic(Intrinsic::cttz, Ops[0]->getType()); in EmitRISCVBuiltinExpr()
19172 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)}); in EmitRISCVBuiltinExpr()
19408 return Builder.CreateCall(F, Ops, ""); in EmitRISCVBuiltinExpr()