Lines Matching refs:CI
1606 static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, in upgradeX86VPERMT2Intrinsics() argument
1608 Type *Ty = CI.getType(); in upgradeX86VPERMT2Intrinsics()
1652 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1), in upgradeX86VPERMT2Intrinsics()
1653 CI.getArgOperand(2) }; in upgradeX86VPERMT2Intrinsics()
1659 Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID), in upgradeX86VPERMT2Intrinsics()
1662 : Builder.CreateBitCast(CI.getArgOperand(1), in upgradeX86VPERMT2Intrinsics()
1664 return emitX86Select(Builder, CI.getArgOperand(3), V, PassThru); in upgradeX86VPERMT2Intrinsics()
1667 static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, in upgradeX86BinaryIntrinsics() argument
1669 Type *Ty = CI.getType(); in upgradeX86BinaryIntrinsics()
1670 Value *Op0 = CI.getOperand(0); in upgradeX86BinaryIntrinsics()
1671 Value *Op1 = CI.getOperand(1); in upgradeX86BinaryIntrinsics()
1672 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in upgradeX86BinaryIntrinsics()
1675 if (CI.arg_size() == 4) { // For masked intrinsics. in upgradeX86BinaryIntrinsics()
1676 Value *VecSrc = CI.getOperand(2); in upgradeX86BinaryIntrinsics()
1677 Value *Mask = CI.getOperand(3); in upgradeX86BinaryIntrinsics()
1683 static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, in upgradeX86Rotate() argument
1685 Type *Ty = CI.getType(); in upgradeX86Rotate()
1686 Value *Src = CI.getArgOperand(0); in upgradeX86Rotate()
1687 Value *Amt = CI.getArgOperand(1); in upgradeX86Rotate()
1699 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in upgradeX86Rotate()
1702 if (CI.arg_size() == 4) { // For masked intrinsics. in upgradeX86Rotate()
1703 Value *VecSrc = CI.getOperand(2); in upgradeX86Rotate()
1704 Value *Mask = CI.getOperand(3); in upgradeX86Rotate()
1710 static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, in upgradeX86vpcom() argument
1712 Type *Ty = CI.getType(); in upgradeX86vpcom()
1713 Value *LHS = CI.getArgOperand(0); in upgradeX86vpcom()
1714 Value *RHS = CI.getArgOperand(1); in upgradeX86vpcom()
1749 static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, in upgradeX86ConcatShift() argument
1751 Type *Ty = CI.getType(); in upgradeX86ConcatShift()
1752 Value *Op0 = CI.getArgOperand(0); in upgradeX86ConcatShift()
1753 Value *Op1 = CI.getArgOperand(1); in upgradeX86ConcatShift()
1754 Value *Amt = CI.getArgOperand(2); in upgradeX86ConcatShift()
1769 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in upgradeX86ConcatShift()
1772 unsigned NumArgs = CI.arg_size(); in upgradeX86ConcatShift()
1774 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) : in upgradeX86ConcatShift()
1775 ZeroMask ? ConstantAggregateZero::get(CI.getType()) : in upgradeX86ConcatShift()
1776 CI.getArgOperand(0); in upgradeX86ConcatShift()
1777 Value *Mask = CI.getOperand(NumArgs - 1); in upgradeX86ConcatShift()
1827 static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) { in upgradeAbs() argument
1828 Type *Ty = CI.getType(); in upgradeAbs()
1829 Value *Op0 = CI.getArgOperand(0); in upgradeAbs()
1830 Function *F = Intrinsic::getDeclaration(CI.getModule(), Intrinsic::abs, Ty); in upgradeAbs()
1832 if (CI.arg_size() == 3) in upgradeAbs()
1833 Res = emitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1)); in upgradeAbs()
1837 static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) { in upgradePMULDQ() argument
1838 Type *Ty = CI.getType(); in upgradePMULDQ()
1841 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty); in upgradePMULDQ()
1842 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty); in upgradePMULDQ()
1860 if (CI.arg_size() == 4) in upgradePMULDQ()
1861 Res = emitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2)); in upgradePMULDQ()
1889 static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, in upgradeMaskedCompare() argument
1891 Value *Op0 = CI.getArgOperand(0); in upgradeMaskedCompare()
1912 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1)); in upgradeMaskedCompare()
1915 Value *Mask = CI.getArgOperand(CI.arg_size() - 1); in upgradeMaskedCompare()
1921 static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, in upgradeX86MaskedShift() argument
1923 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID); in upgradeX86MaskedShift()
1925 { CI.getArgOperand(0), CI.getArgOperand(1) }); in upgradeX86MaskedShift()
1926 return emitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2)); in upgradeX86MaskedShift()
1929 static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) { in upgradeMaskedMove() argument
1930 Value* A = CI.getArgOperand(0); in upgradeMaskedMove()
1931 Value* B = CI.getArgOperand(1); in upgradeMaskedMove()
1932 Value* Src = CI.getArgOperand(2); in upgradeMaskedMove()
1933 Value* Mask = CI.getArgOperand(3); in upgradeMaskedMove()
1943 static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) { in upgradeMaskToInt() argument
1944 Value* Op = CI.getArgOperand(0); in upgradeMaskToInt()
1945 Type* ReturnOp = CI.getType(); in upgradeMaskToInt()
1946 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements(); in upgradeMaskToInt()
1953 CallBase &CI, Value *&Rep) { in upgradeAVX512MaskToSelect() argument
1956 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits(); in upgradeAVX512MaskToSelect()
1957 unsigned EltWidth = CI.getType()->getScalarSizeInBits(); in upgradeAVX512MaskToSelect()
2097 bool IsFloat = CI.getType()->isFPOrFPVectorTy(); in upgradeAVX512MaskToSelect()
2179 SmallVector<Value *, 4> Args(CI.args()); in upgradeAVX512MaskToSelect()
2182 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID), in upgradeAVX512MaskToSelect()
2184 unsigned NumArgs = CI.arg_size(); in upgradeAVX512MaskToSelect()
2185 Rep = emitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep, in upgradeAVX512MaskToSelect()
2186 CI.getArgOperand(NumArgs - 2)); in upgradeAVX512MaskToSelect()
2201 static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, in upgradeARMIntrinsicCall() argument
2208 CI->getArgOperand(0), CI->getName()); in upgradeARMIntrinsicCall()
2238 unsigned ID = CI->getIntrinsicID(); in upgradeARMIntrinsicCall()
2244 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty}; in upgradeARMIntrinsicCall()
2249 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(), in upgradeARMIntrinsicCall()
2253 Tys = {CI->getType(), CI->getOperand(0)->getType(), in upgradeARMIntrinsicCall()
2254 CI->getOperand(1)->getType(), V2I1Ty}; in upgradeARMIntrinsicCall()
2257 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(), in upgradeARMIntrinsicCall()
2258 CI->getOperand(2)->getType(), V2I1Ty}; in upgradeARMIntrinsicCall()
2266 Tys = {CI->getOperand(1)->getType(), V2I1Ty}; in upgradeARMIntrinsicCall()
2273 for (Value *Op : CI->args()) { in upgradeARMIntrinsicCall()
2290 return Builder.CreateCall(Fn, Ops, CI->getName()); in upgradeARMIntrinsicCall()
2295 static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, in upgradeAMDGCNIntrinsicCall() argument
2299 if (CI->getNumOperands() != 6) // Malformed bitcode. in upgradeAMDGCNIntrinsicCall()
2305 Value *Ptr = CI->getArgOperand(0); in upgradeAMDGCNIntrinsicCall()
2306 Value *Val = CI->getArgOperand(1); in upgradeAMDGCNIntrinsicCall()
2307 ConstantInt *OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2)); in upgradeAMDGCNIntrinsicCall()
2308 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4)); in upgradeAMDGCNIntrinsicCall()
2333 void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) { in UpgradeIntrinsicCall() argument
2337 Function *F = dyn_cast<Function>(CI->getCalledOperand()); in UpgradeIntrinsicCall()
2341 LLVMContext &C = CI->getContext(); in UpgradeIntrinsicCall()
2343 Builder.SetInsertPoint(CI->getParent(), CI->getIterator()); in UpgradeIntrinsicCall()
2363 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2364 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
2378 CI->eraseFromParent(); in UpgradeIntrinsicCall()
2389 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2390 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
2402 CI->eraseFromParent(); in UpgradeIntrinsicCall()
2407 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2408 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
2419 CI->eraseFromParent(); in UpgradeIntrinsicCall()
2426 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2427 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
2435 CI->eraseFromParent(); in UpgradeIntrinsicCall()
2440 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1)); in UpgradeIntrinsicCall()
2441 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
2445 CI->eraseFromParent(); in UpgradeIntrinsicCall()
2452 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
2453 CI->getArgOperand(2), Aligned); in UpgradeIntrinsicCall()
2456 CI->eraseFromParent(); in UpgradeIntrinsicCall()
2467 CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2468 Rep = Builder.CreateSExt(Rep, CI->getType(), ""); in UpgradeIntrinsicCall()
2471 if (CI->getOperand(0)->getType()->isIntegerTy(8)) in UpgradeIntrinsicCall()
2473 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / in UpgradeIntrinsicCall()
2475 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy); in UpgradeIntrinsicCall()
2479 Value *Vec = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2490 CI->getType()), in UpgradeIntrinsicCall()
2491 {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
2493 if (CI->arg_size() == 4 && in UpgradeIntrinsicCall()
2494 (!isa<ConstantInt>(CI->getArgOperand(3)) || in UpgradeIntrinsicCall()
2495 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) { in UpgradeIntrinsicCall()
2499 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(3) }; in UpgradeIntrinsicCall()
2500 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), in UpgradeIntrinsicCall()
2505 CI->getType()), in UpgradeIntrinsicCall()
2506 {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
2508 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2509 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2512 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2513 Value *Op1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
2514 Value *Mask = CI->getArgOperand(2); in UpgradeIntrinsicCall()
2523 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType()) in UpgradeIntrinsicCall()
2525 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0)); in UpgradeIntrinsicCall()
2526 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2527 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2529 unsigned NumElts = CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
2530 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts); in UpgradeIntrinsicCall()
2531 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts); in UpgradeIntrinsicCall()
2545 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2547 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2548 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in UpgradeIntrinsicCall()
2550 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2552 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2553 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in UpgradeIntrinsicCall()
2556 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2558 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2559 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in UpgradeIntrinsicCall()
2561 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2563 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2564 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in UpgradeIntrinsicCall()
2566 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2568 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2569 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in UpgradeIntrinsicCall()
2572 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2574 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2576 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2579 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2580 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in UpgradeIntrinsicCall()
2595 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0), in UpgradeIntrinsicCall()
2597 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1), in UpgradeIntrinsicCall()
2608 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp, in UpgradeIntrinsicCall()
2613 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true); in UpgradeIntrinsicCall()
2615 Type *OpTy = CI->getArgOperand(0)->getType(); in UpgradeIntrinsicCall()
2626 { CI->getOperand(0), CI->getArgOperand(1) }); in UpgradeIntrinsicCall()
2627 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2629 Type *OpTy = CI->getArgOperand(0)->getType(); in UpgradeIntrinsicCall()
2649 { CI->getOperand(0), CI->getArgOperand(1) }); in UpgradeIntrinsicCall()
2650 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2652 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
2672 Value *Mask = Constant::getAllOnesValue(CI->getType()); in UpgradeIntrinsicCall()
2681 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
2682 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true); in UpgradeIntrinsicCall()
2684 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
2685 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false); in UpgradeIntrinsicCall()
2690 Value *Op = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2699 Rep = upgradeAbs(Builder, *CI); in UpgradeIntrinsicCall()
2705 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax); in UpgradeIntrinsicCall()
2711 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax); in UpgradeIntrinsicCall()
2717 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin); in UpgradeIntrinsicCall()
2723 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin); in UpgradeIntrinsicCall()
2728 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/false); in UpgradeIntrinsicCall()
2733 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/true); in UpgradeIntrinsicCall()
2739 CI->getArgOperand(1), in UpgradeIntrinsicCall()
2740 cast<VectorType>(CI->getType())->getElementType()); in UpgradeIntrinsicCall()
2741 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); in UpgradeIntrinsicCall()
2744 CI->getArgOperand(1), in UpgradeIntrinsicCall()
2745 cast<VectorType>(CI->getType())->getElementType()); in UpgradeIntrinsicCall()
2746 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); in UpgradeIntrinsicCall()
2748 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0); in UpgradeIntrinsicCall()
2750 Rep, cast<VectorType>(CI->getType())->getElementType()); in UpgradeIntrinsicCall()
2751 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); in UpgradeIntrinsicCall()
2770 auto *DstTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2771 Rep = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2784 else if (CI->arg_size() == 4 && in UpgradeIntrinsicCall()
2785 (!isa<ConstantInt>(CI->getArgOperand(3)) || in UpgradeIntrinsicCall()
2786 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) { in UpgradeIntrinsicCall()
2789 Function *F = Intrinsic::getDeclaration(CI->getModule(), IID, in UpgradeIntrinsicCall()
2791 Rep = Builder.CreateCall(F, { Rep, CI->getArgOperand(3) }); in UpgradeIntrinsicCall()
2797 if (CI->arg_size() >= 3) in UpgradeIntrinsicCall()
2798 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2799 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2802 auto *DstTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2803 Rep = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2813 if (CI->arg_size() >= 3) in UpgradeIntrinsicCall()
2814 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2815 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2820 upgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
2821 CI->getArgOperand(2), Aligned); in UpgradeIntrinsicCall()
2823 auto *ResultTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2827 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0), in UpgradeIntrinsicCall()
2830 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2), in UpgradeIntrinsicCall()
2836 Rep = Builder.CreateCall(ELd, { Ptr, MaskVec, CI->getOperand(1) }); in UpgradeIntrinsicCall()
2838 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType()); in UpgradeIntrinsicCall()
2842 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0), in UpgradeIntrinsicCall()
2846 getX86MaskVec(Builder, CI->getArgOperand(2), in UpgradeIntrinsicCall()
2852 Rep = Builder.CreateCall(CSt, { CI->getArgOperand(1), Ptr, MaskVec }); in UpgradeIntrinsicCall()
2855 auto *ResultTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2857 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2), in UpgradeIntrinsicCall()
2864 Rep = Builder.CreateCall(Intr, { CI->getOperand(0), CI->getOperand(1), in UpgradeIntrinsicCall()
2878 if (CI->arg_size() == 3) { in UpgradeIntrinsicCall()
2879 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
2902 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned); in UpgradeIntrinsicCall()
2904 Value *Sel = CI->getArgOperand(2); in UpgradeIntrinsicCall()
2906 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel); in UpgradeIntrinsicCall()
2907 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel); in UpgradeIntrinsicCall()
2912 Rep = upgradeX86Rotate(Builder, *CI, false); in UpgradeIntrinsicCall()
2915 Rep = upgradeX86Rotate(Builder, *CI, true); in UpgradeIntrinsicCall()
2920 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask); in UpgradeIntrinsicCall()
2925 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask); in UpgradeIntrinsicCall()
2929 Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C)); in UpgradeIntrinsicCall()
2930 Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)}); in UpgradeIntrinsicCall()
2931 Rep = Builder.CreateZExt(Rep, CI->getType(), ""); in UpgradeIntrinsicCall()
2935 auto *VecTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2938 Value *Load = Builder.CreateLoad(EltTy, CI->getArgOperand(0)); in UpgradeIntrinsicCall()
2950 auto *DstTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2959 Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask); in UpgradeIntrinsicCall()
2965 if (CI->arg_size() == 3) in UpgradeIntrinsicCall()
2966 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2967 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2972 Type *Ty = CI->getArgOperand(1)->getType(); in UpgradeIntrinsicCall()
2973 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty); in UpgradeIntrinsicCall()
2974 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2975 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2979 Type *EltTy = cast<VectorType>(CI->getType())->getElementType(); in UpgradeIntrinsicCall()
2982 Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0), in UpgradeIntrinsicCall()
2992 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
2993 Type *VT = CI->getType(); in UpgradeIntrinsicCall()
3008 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0), in UpgradeIntrinsicCall()
3009 CI->getArgOperand(1), ShuffleMask); in UpgradeIntrinsicCall()
3010 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, in UpgradeIntrinsicCall()
3011 CI->getArgOperand(3)); in UpgradeIntrinsicCall()
3015 cast<FixedVectorType>(CI->getArgOperand(0)->getType()) in UpgradeIntrinsicCall()
3018 cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
3024 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0), in UpgradeIntrinsicCall()
3025 CI->getArgOperand(0), in UpgradeIntrinsicCall()
3027 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
3028 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
3034 Value *Op = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3035 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount(); in UpgradeIntrinsicCall()
3041 if (CI->arg_size() == 3) in UpgradeIntrinsicCall()
3042 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
3043 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
3048 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat); in UpgradeIntrinsicCall()
3053 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat); in UpgradeIntrinsicCall()
3057 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat); in UpgradeIntrinsicCall()
3061 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat); in UpgradeIntrinsicCall()
3064 Builder, CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3065 CI->getArgOperand(2), CI->getArgOperand(3), CI->getArgOperand(4), in UpgradeIntrinsicCall()
3069 Builder, CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3070 CI->getArgOperand(2), CI->getArgOperand(3), CI->getArgOperand(4), in UpgradeIntrinsicCall()
3075 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
3076 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), in UpgradeIntrinsicCall()
3081 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
3082 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), in UpgradeIntrinsicCall()
3088 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
3089 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift); in UpgradeIntrinsicCall()
3094 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
3095 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift); in UpgradeIntrinsicCall()
3101 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3102 Value *Op1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
3103 unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
3104 auto *VecTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
3115 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3116 Value *Op1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
3117 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
3119 cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
3155 if (CI->arg_size() == 5) in UpgradeIntrinsicCall()
3156 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, in UpgradeIntrinsicCall()
3157 CI->getArgOperand(3)); in UpgradeIntrinsicCall()
3161 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3162 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
3164 cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
3180 if (CI->arg_size() == 4) in UpgradeIntrinsicCall()
3181 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3182 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3187 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3188 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
3189 auto *VecTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
3198 if (CI->arg_size() == 4) in UpgradeIntrinsicCall()
3199 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3200 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3211 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
3213 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
3218 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0); in UpgradeIntrinsicCall()
3219 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0); in UpgradeIntrinsicCall()
3222 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0; in UpgradeIntrinsicCall()
3223 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1; in UpgradeIntrinsicCall()
3241 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3242 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
3243 auto *VecTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
3258 if (CI->arg_size() == 4) in UpgradeIntrinsicCall()
3259 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3260 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3263 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3264 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
3265 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
3277 if (CI->arg_size() == 4) in UpgradeIntrinsicCall()
3278 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3279 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3282 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3283 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
3284 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
3296 if (CI->arg_size() == 4) in UpgradeIntrinsicCall()
3297 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3298 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3300 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3301 Value *Op1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
3302 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
3303 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
3305 unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
3322 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, in UpgradeIntrinsicCall()
3323 CI->getArgOperand(3)); in UpgradeIntrinsicCall()
3327 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3328 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
3329 unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
3344 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
3345 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
3348 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3349 Value *Op1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
3350 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
3351 int NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
3360 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3361 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3364 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3365 Value *Op1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
3366 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
3367 int NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
3376 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3377 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3380 VectorType *FTy = cast<VectorType>(CI->getType()); in UpgradeIntrinsicCall()
3382 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy), in UpgradeIntrinsicCall()
3383 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); in UpgradeIntrinsicCall()
3385 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3386 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3389 VectorType *FTy = cast<VectorType>(CI->getType()); in UpgradeIntrinsicCall()
3391 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy)); in UpgradeIntrinsicCall()
3393 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); in UpgradeIntrinsicCall()
3395 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3396 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3399 VectorType *FTy = cast<VectorType>(CI->getType()); in UpgradeIntrinsicCall()
3401 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy), in UpgradeIntrinsicCall()
3402 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); in UpgradeIntrinsicCall()
3404 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3405 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3408 VectorType *FTy = cast<VectorType>(CI->getType()); in UpgradeIntrinsicCall()
3410 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy), in UpgradeIntrinsicCall()
3411 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); in UpgradeIntrinsicCall()
3413 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3414 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3416 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
3417 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3418 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3420 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
3421 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3422 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3424 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
3425 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3426 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3436 { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3437 CI->getArgOperand(4) }); in UpgradeIntrinsicCall()
3439 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
3441 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3442 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3452 { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3453 CI->getArgOperand(4) }); in UpgradeIntrinsicCall()
3455 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
3457 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3458 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3468 { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3469 CI->getArgOperand(4) }); in UpgradeIntrinsicCall()
3471 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
3473 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3474 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3484 { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3485 CI->getArgOperand(4) }); in UpgradeIntrinsicCall()
3487 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
3489 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3490 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3503 { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3504 CI->getArgOperand(4) }); in UpgradeIntrinsicCall()
3505 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3506 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3510 CI->getType()), in UpgradeIntrinsicCall()
3511 { CI->getArgOperand(0), Builder.getInt1(false) }); in UpgradeIntrinsicCall()
3512 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
3513 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
3581 Rep = upgradeX86MaskedShift(Builder, *CI, IID); in UpgradeIntrinsicCall()
3649 Rep = upgradeX86MaskedShift(Builder, *CI, IID); in UpgradeIntrinsicCall()
3715 Rep = upgradeX86MaskedShift(Builder, *CI, IID); in UpgradeIntrinsicCall()
3717 Rep = upgradeMaskedMove(Builder, *CI); in UpgradeIntrinsicCall()
3719 Rep = upgradeMaskToInt(Builder, *CI); in UpgradeIntrinsicCall()
3724 Value *Ptr = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3728 Ptr, PointerType::getUnqual(CI->getType()), "cast"); in UpgradeIntrinsicCall()
3730 CI->getType(), BC, in UpgradeIntrinsicCall()
3731 Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)); in UpgradeIntrinsicCall()
3742 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3743 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
3758 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), in UpgradeIntrinsicCall()
3764 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, in UpgradeIntrinsicCall()
3767 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3768 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
3774 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), in UpgradeIntrinsicCall()
3779 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()), in UpgradeIntrinsicCall()
3793 Value *A = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3794 Value *B = CI->getArgOperand(1); in UpgradeIntrinsicCall()
3795 Value *C = CI->getArgOperand(2); in UpgradeIntrinsicCall()
3808 if (!isa<ConstantInt>(CI->getArgOperand(4)) || in UpgradeIntrinsicCall()
3809 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) { in UpgradeIntrinsicCall()
3810 Value *Ops[] = { A, B, C, CI->getArgOperand(4) }; in UpgradeIntrinsicCall()
3817 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID); in UpgradeIntrinsicCall()
3820 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), in UpgradeIntrinsicCall()
3832 PassThru = Builder.CreateExtractElement(CI->getArgOperand(2), in UpgradeIntrinsicCall()
3835 Rep = emitX86ScalarSelect(Builder, CI->getArgOperand(3), Rep, PassThru); in UpgradeIntrinsicCall()
3836 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), in UpgradeIntrinsicCall()
3852 Value *A = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3853 Value *B = CI->getArgOperand(1); in UpgradeIntrinsicCall()
3854 Value *C = CI->getArgOperand(2); in UpgradeIntrinsicCall()
3863 if (CI->arg_size() == 5 && in UpgradeIntrinsicCall()
3864 (!isa<ConstantInt>(CI->getArgOperand(4)) || in UpgradeIntrinsicCall()
3865 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) { in UpgradeIntrinsicCall()
3874 { A, B, C, CI->getArgOperand(4) }); in UpgradeIntrinsicCall()
3876 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), in UpgradeIntrinsicCall()
3882 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) : in UpgradeIntrinsicCall()
3883 IsMask3 ? CI->getArgOperand(2) : in UpgradeIntrinsicCall()
3884 CI->getArgOperand(0); in UpgradeIntrinsicCall()
3886 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in UpgradeIntrinsicCall()
3888 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in UpgradeIntrinsicCall()
3889 unsigned EltWidth = CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
3902 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3903 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
3916 if (CI->arg_size() == 5) { in UpgradeIntrinsicCall()
3924 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3925 CI->getArgOperand(2), CI->getArgOperand(4) }; in UpgradeIntrinsicCall()
3932 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
3934 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3935 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
3937 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma, in UpgradeIntrinsicCall()
3953 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) : in UpgradeIntrinsicCall()
3954 IsMask3 ? CI->getArgOperand(2) : in UpgradeIntrinsicCall()
3955 CI->getArgOperand(0); in UpgradeIntrinsicCall()
3957 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in UpgradeIntrinsicCall()
3961 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in UpgradeIntrinsicCall()
3962 unsigned EltWidth = CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
3979 Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1), in UpgradeIntrinsicCall()
3980 CI->getArgOperand(2), CI->getArgOperand(3) }; in UpgradeIntrinsicCall()
3981 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
3983 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) in UpgradeIntrinsicCall()
3984 : CI->getArgOperand(0); in UpgradeIntrinsicCall()
3985 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru); in UpgradeIntrinsicCall()
3990 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in UpgradeIntrinsicCall()
4007 Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1), in UpgradeIntrinsicCall()
4008 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
4009 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
4011 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) in UpgradeIntrinsicCall()
4012 : CI->getArgOperand(0); in UpgradeIntrinsicCall()
4013 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in UpgradeIntrinsicCall()
4019 Rep = upgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm); in UpgradeIntrinsicCall()
4026 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in UpgradeIntrinsicCall()
4043 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
4044 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
4045 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
4047 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) in UpgradeIntrinsicCall()
4048 : CI->getArgOperand(0); in UpgradeIntrinsicCall()
4049 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in UpgradeIntrinsicCall()
4056 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in UpgradeIntrinsicCall()
4073 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
4074 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
4075 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
4077 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) in UpgradeIntrinsicCall()
4078 : CI->getArgOperand(0); in UpgradeIntrinsicCall()
4079 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in UpgradeIntrinsicCall()
4096 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
4097 CI->getArgOperand(2)}; in UpgradeIntrinsicCall()
4099 Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
4105 Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(3), in UpgradeIntrinsicCall()
4111 CI->replaceAllUsesWith(CF); in UpgradeIntrinsicCall()
4114 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) { in UpgradeIntrinsicCall()
4117 Value *Arg = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4124 Value *Ptr = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4125 Value *Val = CI->getArgOperand(1); in UpgradeIntrinsicCall()
4131 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4132 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
4140 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4141 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
4148 Value *Arg = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4157 Value *Arg = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4169 CI->getArgOperand(0), "h2f"); in UpgradeIntrinsicCall()
4178 Value *Arg = CI->getArgOperand(I); in UpgradeIntrinsicCall()
4192 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder); in UpgradeIntrinsicCall()
4194 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder); in UpgradeIntrinsicCall()
4200 CI->replaceAllUsesWith(Rep); in UpgradeIntrinsicCall()
4201 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4206 if (CI->getFunctionType() == NewFn->getFunctionType()) { in UpgradeIntrinsicCall()
4209 (CI->getCalledFunction()->getName() != NewFn->getName()) && in UpgradeIntrinsicCall()
4211 CI->setCalledFunction(NewFn); in UpgradeIntrinsicCall()
4216 if (auto *OldST = dyn_cast<StructType>(CI->getType())) { in UpgradeIntrinsicCall()
4223 SmallVector<Value *> Args(CI->args()); in UpgradeIntrinsicCall()
4230 CI->replaceAllUsesWith(Res); in UpgradeIntrinsicCall()
4231 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4237 CI->setCalledOperand( in UpgradeIntrinsicCall()
4238 ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType())); in UpgradeIntrinsicCall()
4254 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4262 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4281 SmallVector<Value *, 2> Args(CI->args()); in UpgradeIntrinsicCall()
4294 SmallVector<Value *, 3> Args(CI->args()); in UpgradeIntrinsicCall()
4295 Args.push_back(ConstantTokenNone::get(CI->getContext())); in UpgradeIntrinsicCall()
4310 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
4312 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx}); in UpgradeIntrinsicCall()
4324 unsigned I = dyn_cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
4326 dyn_cast<ScalableVectorType>(CI->getArgOperand(2)->getType()); in UpgradeIntrinsicCall()
4330 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx}); in UpgradeIntrinsicCall()
4346 Value *V = CI->getArgOperand(I); in UpgradeIntrinsicCall()
4363 assert(CI->arg_size() == 3 && in UpgradeIntrinsicCall()
4366 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits(); in UpgradeIntrinsicCall()
4370 auto Iter = CI->args().begin(); in UpgradeIntrinsicCall()
4379 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
4384 assert(CI->arg_size() == 1 && in UpgradeIntrinsicCall()
4387 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()}); in UpgradeIntrinsicCall()
4392 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2); in UpgradeIntrinsicCall()
4394 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3); in UpgradeIntrinsicCall()
4396 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic}); in UpgradeIntrinsicCall()
4401 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
4405 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
4414 cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata()); in UpgradeIntrinsicCall()
4417 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
4423 assert(CI->arg_size() == 4); in UpgradeIntrinsicCall()
4425 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1))) in UpgradeIntrinsicCall()
4429 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)}); in UpgradeIntrinsicCall()
4432 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4438 if (CI->arg_size() != 4) { in UpgradeIntrinsicCall()
4445 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
4446 CI->getArgOperand(2), CI->getArgOperand(3), in UpgradeIntrinsicCall()
4448 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4449 CI->replaceAllUsesWith(NewCall); in UpgradeIntrinsicCall()
4450 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4455 if (CI->arg_size() != 4) { in UpgradeIntrinsicCall()
4461 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
4462 CI->getArgOperand(2), CI->getArgOperand(3), in UpgradeIntrinsicCall()
4464 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4465 CI->replaceAllUsesWith(NewCall); in UpgradeIntrinsicCall()
4466 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4477 Value *Arg2 = CI->getArgOperand(2); in UpgradeIntrinsicCall()
4478 if (Arg2->getType()->isIntegerTy(32) && !CI->getType()->isIntegerTy(64)) in UpgradeIntrinsicCall()
4481 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4482 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
4483 if (CI->getType()->isIntegerTy(64)) { in UpgradeIntrinsicCall()
4493 if (Res->getType() != CI->getType()) in UpgradeIntrinsicCall()
4494 Res = Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true); in UpgradeIntrinsicCall()
4495 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4496 CI->replaceAllUsesWith(Res); in UpgradeIntrinsicCall()
4497 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4508 if (!CI->getType()->isIntegerTy(64)) in UpgradeIntrinsicCall()
4512 Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty()); in UpgradeIntrinsicCall()
4516 Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true); in UpgradeIntrinsicCall()
4517 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4518 CI->replaceAllUsesWith(Res); in UpgradeIntrinsicCall()
4519 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4525 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)}); in UpgradeIntrinsicCall()
4532 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4547 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4552 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
4566 if (CI->getNumOperands() == 0) in UpgradeIntrinsicCall()
4573 Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(0), in UpgradeIntrinsicCall()
4579 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4580 CI->replaceAllUsesWith(TSC); in UpgradeIntrinsicCall()
4581 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4593 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4607 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4615 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4616 CI->replaceAllUsesWith(Res); in UpgradeIntrinsicCall()
4617 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4627 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4628 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
4638 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4639 CI->replaceAllUsesWith(Res); in UpgradeIntrinsicCall()
4640 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4646 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4648 cast<FixedVectorType>(CI->getType())->getNumElements() * 2; in UpgradeIntrinsicCall()
4673 if (CI->arg_size() != 5) { in UpgradeIntrinsicCall()
4679 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
4680 CI->getArgOperand(2), CI->getArgOperand(4)}; in UpgradeIntrinsicCall()
4682 AttributeList OldAttrs = CI->getAttributes(); in UpgradeIntrinsicCall()
4690 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3)); in UpgradeIntrinsicCall()
4700 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4701 CI->replaceAllUsesWith(NewCall); in UpgradeIntrinsicCall()
4702 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4859 CallInst *CI = dyn_cast<CallInst>(U); in UpgradeARCRuntime() local
4860 if (!CI || CI->getCalledFunction() != Fn) in UpgradeARCRuntime()
4863 IRBuilder<> Builder(CI->getParent(), CI->getIterator()); in UpgradeARCRuntime()
4869 if (NewFuncTy->getReturnType() != CI->getType() && in UpgradeARCRuntime()
4870 !CastInst::castIsValid(Instruction::BitCast, CI, in UpgradeARCRuntime()
4876 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) { in UpgradeARCRuntime()
4877 Value *Arg = CI->getArgOperand(I); in UpgradeARCRuntime()
4899 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind()); in UpgradeARCRuntime()
4900 NewCall->takeName(CI); in UpgradeARCRuntime()
4903 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType()); in UpgradeARCRuntime()
4905 if (!CI->use_empty()) in UpgradeARCRuntime()
4906 CI->replaceAllUsesWith(NewRetVal); in UpgradeARCRuntime()
4907 CI->eraseFromParent(); in UpgradeARCRuntime()