Lines Matching refs:CI
1207 static Value *UpgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallInst &CI, in UpgradeX86VPERMT2Intrinsics() argument
1209 Type *Ty = CI.getType(); in UpgradeX86VPERMT2Intrinsics()
1253 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1), in UpgradeX86VPERMT2Intrinsics()
1254 CI.getArgOperand(2) }; in UpgradeX86VPERMT2Intrinsics()
1260 Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID), in UpgradeX86VPERMT2Intrinsics()
1263 : Builder.CreateBitCast(CI.getArgOperand(1), in UpgradeX86VPERMT2Intrinsics()
1265 return EmitX86Select(Builder, CI.getArgOperand(3), V, PassThru); in UpgradeX86VPERMT2Intrinsics()
1268 static Value *UpgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallInst &CI, in UpgradeX86BinaryIntrinsics() argument
1270 Type *Ty = CI.getType(); in UpgradeX86BinaryIntrinsics()
1271 Value *Op0 = CI.getOperand(0); in UpgradeX86BinaryIntrinsics()
1272 Value *Op1 = CI.getOperand(1); in UpgradeX86BinaryIntrinsics()
1273 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in UpgradeX86BinaryIntrinsics()
1276 if (CI.getNumArgOperands() == 4) { // For masked intrinsics. in UpgradeX86BinaryIntrinsics()
1277 Value *VecSrc = CI.getOperand(2); in UpgradeX86BinaryIntrinsics()
1278 Value *Mask = CI.getOperand(3); in UpgradeX86BinaryIntrinsics()
1284 static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallInst &CI, in upgradeX86Rotate() argument
1286 Type *Ty = CI.getType(); in upgradeX86Rotate()
1287 Value *Src = CI.getArgOperand(0); in upgradeX86Rotate()
1288 Value *Amt = CI.getArgOperand(1); in upgradeX86Rotate()
1300 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in upgradeX86Rotate()
1303 if (CI.getNumArgOperands() == 4) { // For masked intrinsics. in upgradeX86Rotate()
1304 Value *VecSrc = CI.getOperand(2); in upgradeX86Rotate()
1305 Value *Mask = CI.getOperand(3); in upgradeX86Rotate()
1311 static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallInst &CI, unsigned Imm, in upgradeX86vpcom() argument
1313 Type *Ty = CI.getType(); in upgradeX86vpcom()
1314 Value *LHS = CI.getArgOperand(0); in upgradeX86vpcom()
1315 Value *RHS = CI.getArgOperand(1); in upgradeX86vpcom()
1350 static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallInst &CI, in upgradeX86ConcatShift() argument
1352 Type *Ty = CI.getType(); in upgradeX86ConcatShift()
1353 Value *Op0 = CI.getArgOperand(0); in upgradeX86ConcatShift()
1354 Value *Op1 = CI.getArgOperand(1); in upgradeX86ConcatShift()
1355 Value *Amt = CI.getArgOperand(2); in upgradeX86ConcatShift()
1370 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in upgradeX86ConcatShift()
1373 unsigned NumArgs = CI.getNumArgOperands(); in upgradeX86ConcatShift()
1375 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) : in upgradeX86ConcatShift()
1376 ZeroMask ? ConstantAggregateZero::get(CI.getType()) : in upgradeX86ConcatShift()
1377 CI.getArgOperand(0); in upgradeX86ConcatShift()
1378 Value *Mask = CI.getOperand(NumArgs - 1); in upgradeX86ConcatShift()
1429 static Value *upgradeAbs(IRBuilder<> &Builder, CallInst &CI) { in upgradeAbs() argument
1430 Type *Ty = CI.getType(); in upgradeAbs()
1431 Value *Op0 = CI.getArgOperand(0); in upgradeAbs()
1432 Function *F = Intrinsic::getDeclaration(CI.getModule(), Intrinsic::abs, Ty); in upgradeAbs()
1434 if (CI.getNumArgOperands() == 3) in upgradeAbs()
1435 Res = EmitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1)); in upgradeAbs()
1439 static Value *upgradePMULDQ(IRBuilder<> &Builder, CallInst &CI, bool IsSigned) { in upgradePMULDQ() argument
1440 Type *Ty = CI.getType(); in upgradePMULDQ()
1443 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty); in upgradePMULDQ()
1444 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty); in upgradePMULDQ()
1462 if (CI.getNumArgOperands() == 4) in upgradePMULDQ()
1463 Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2)); in upgradePMULDQ()
1491 static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallInst &CI, in upgradeMaskedCompare() argument
1493 Value *Op0 = CI.getArgOperand(0); in upgradeMaskedCompare()
1514 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1)); in upgradeMaskedCompare()
1517 Value *Mask = CI.getArgOperand(CI.getNumArgOperands() - 1); in upgradeMaskedCompare()
1523 static Value *UpgradeX86MaskedShift(IRBuilder<> &Builder, CallInst &CI, in UpgradeX86MaskedShift() argument
1525 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID); in UpgradeX86MaskedShift()
1527 { CI.getArgOperand(0), CI.getArgOperand(1) }); in UpgradeX86MaskedShift()
1528 return EmitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2)); in UpgradeX86MaskedShift()
1531 static Value* upgradeMaskedMove(IRBuilder<> &Builder, CallInst &CI) { in upgradeMaskedMove() argument
1532 Value* A = CI.getArgOperand(0); in upgradeMaskedMove()
1533 Value* B = CI.getArgOperand(1); in upgradeMaskedMove()
1534 Value* Src = CI.getArgOperand(2); in upgradeMaskedMove()
1535 Value* Mask = CI.getArgOperand(3); in upgradeMaskedMove()
1546 static Value* UpgradeMaskToInt(IRBuilder<> &Builder, CallInst &CI) { in UpgradeMaskToInt() argument
1547 Value* Op = CI.getArgOperand(0); in UpgradeMaskToInt()
1548 Type* ReturnOp = CI.getType(); in UpgradeMaskToInt()
1549 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements(); in UpgradeMaskToInt()
1556 CallInst &CI, Value *&Rep) { in upgradeAVX512MaskToSelect() argument
1559 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits(); in upgradeAVX512MaskToSelect()
1560 unsigned EltWidth = CI.getType()->getScalarSizeInBits(); in upgradeAVX512MaskToSelect()
1700 bool IsFloat = CI.getType()->isFPOrFPVectorTy(); in upgradeAVX512MaskToSelect()
1782 SmallVector<Value *, 4> Args(CI.arg_operands().begin(), in upgradeAVX512MaskToSelect()
1783 CI.arg_operands().end()); in upgradeAVX512MaskToSelect()
1786 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID), in upgradeAVX512MaskToSelect()
1788 unsigned NumArgs = CI.getNumArgOperands(); in upgradeAVX512MaskToSelect()
1789 Rep = EmitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep, in upgradeAVX512MaskToSelect()
1790 CI.getArgOperand(NumArgs - 2)); in upgradeAVX512MaskToSelect()
1807 void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { in UpgradeIntrinsicCall() argument
1808 Function *F = CI->getCalledFunction(); in UpgradeIntrinsicCall()
1809 LLVMContext &C = CI->getContext(); in UpgradeIntrinsicCall()
1811 Builder.SetInsertPoint(CI->getParent(), CI->getIterator()); in UpgradeIntrinsicCall()
1836 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
1837 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
1851 CI->eraseFromParent(); in UpgradeIntrinsicCall()
1863 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
1864 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
1876 CI->eraseFromParent(); in UpgradeIntrinsicCall()
1881 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
1882 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
1893 CI->eraseFromParent(); in UpgradeIntrinsicCall()
1900 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
1901 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
1909 CI->eraseFromParent(); in UpgradeIntrinsicCall()
1914 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1)); in UpgradeIntrinsicCall()
1915 UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
1919 CI->eraseFromParent(); in UpgradeIntrinsicCall()
1926 UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
1927 CI->getArgOperand(2), Aligned); in UpgradeIntrinsicCall()
1930 CI->eraseFromParent(); in UpgradeIntrinsicCall()
1941 CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
1942 Rep = Builder.CreateSExt(Rep, CI->getType(), ""); in UpgradeIntrinsicCall()
1945 if (CI->getOperand(0)->getType()->isIntegerTy(8)) in UpgradeIntrinsicCall()
1947 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / in UpgradeIntrinsicCall()
1949 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy); in UpgradeIntrinsicCall()
1953 Value *Vec = CI->getArgOperand(0); in UpgradeIntrinsicCall()
1964 CI->getType()), in UpgradeIntrinsicCall()
1965 {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
1967 if (CI->getNumArgOperands() == 4 && in UpgradeIntrinsicCall()
1968 (!isa<ConstantInt>(CI->getArgOperand(3)) || in UpgradeIntrinsicCall()
1969 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) { in UpgradeIntrinsicCall()
1973 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(3) }; in UpgradeIntrinsicCall()
1974 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), in UpgradeIntrinsicCall()
1979 CI->getType()), in UpgradeIntrinsicCall()
1980 {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
1982 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
1983 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
1986 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
1987 Value *Op1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
1988 Value *Mask = CI->getArgOperand(2); in UpgradeIntrinsicCall()
1997 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType()) in UpgradeIntrinsicCall()
1999 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0)); in UpgradeIntrinsicCall()
2000 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2001 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2003 unsigned NumElts = CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
2004 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts); in UpgradeIntrinsicCall()
2005 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts); in UpgradeIntrinsicCall()
2020 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2022 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2023 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in UpgradeIntrinsicCall()
2025 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2027 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2028 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in UpgradeIntrinsicCall()
2031 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2033 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2034 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in UpgradeIntrinsicCall()
2036 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2038 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2039 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in UpgradeIntrinsicCall()
2041 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2043 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2044 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in UpgradeIntrinsicCall()
2047 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2049 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2051 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2054 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2055 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in UpgradeIntrinsicCall()
2070 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0), in UpgradeIntrinsicCall()
2072 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1), in UpgradeIntrinsicCall()
2083 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp, in UpgradeIntrinsicCall()
2088 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true); in UpgradeIntrinsicCall()
2090 Type *OpTy = CI->getArgOperand(0)->getType(); in UpgradeIntrinsicCall()
2101 { CI->getOperand(0), CI->getArgOperand(1) }); in UpgradeIntrinsicCall()
2102 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2104 Type *OpTy = CI->getArgOperand(0)->getType(); in UpgradeIntrinsicCall()
2124 { CI->getOperand(0), CI->getArgOperand(1) }); in UpgradeIntrinsicCall()
2125 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2127 SmallVector<Value *, 4> Args(CI->arg_operands().begin(), in UpgradeIntrinsicCall()
2128 CI->arg_operands().end()); in UpgradeIntrinsicCall()
2148 Value *Mask = Constant::getAllOnesValue(CI->getType()); in UpgradeIntrinsicCall()
2157 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
2158 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true); in UpgradeIntrinsicCall()
2160 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
2161 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false); in UpgradeIntrinsicCall()
2166 Value *Op = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2175 Rep = upgradeAbs(Builder, *CI); in UpgradeIntrinsicCall()
2181 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax); in UpgradeIntrinsicCall()
2187 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax); in UpgradeIntrinsicCall()
2193 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin); in UpgradeIntrinsicCall()
2199 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin); in UpgradeIntrinsicCall()
2204 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/false); in UpgradeIntrinsicCall()
2209 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/true); in UpgradeIntrinsicCall()
2215 CI->getArgOperand(1), in UpgradeIntrinsicCall()
2216 cast<VectorType>(CI->getType())->getElementType()); in UpgradeIntrinsicCall()
2217 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); in UpgradeIntrinsicCall()
2220 CI->getArgOperand(1), in UpgradeIntrinsicCall()
2221 cast<VectorType>(CI->getType())->getElementType()); in UpgradeIntrinsicCall()
2222 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); in UpgradeIntrinsicCall()
2224 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0); in UpgradeIntrinsicCall()
2226 Rep, cast<VectorType>(CI->getType())->getElementType()); in UpgradeIntrinsicCall()
2227 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); in UpgradeIntrinsicCall()
2246 auto *DstTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2247 Rep = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2260 else if (CI->getNumArgOperands() == 4 && in UpgradeIntrinsicCall()
2261 (!isa<ConstantInt>(CI->getArgOperand(3)) || in UpgradeIntrinsicCall()
2262 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) { in UpgradeIntrinsicCall()
2265 Function *F = Intrinsic::getDeclaration(CI->getModule(), IID, in UpgradeIntrinsicCall()
2267 Rep = Builder.CreateCall(F, { Rep, CI->getArgOperand(3) }); in UpgradeIntrinsicCall()
2273 if (CI->getNumArgOperands() >= 3) in UpgradeIntrinsicCall()
2274 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2275 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2278 auto *DstTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2279 Rep = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2289 if (CI->getNumArgOperands() >= 3) in UpgradeIntrinsicCall()
2290 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2291 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2293 Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0), in UpgradeIntrinsicCall()
2294 CI->getArgOperand(1), CI->getArgOperand(2), in UpgradeIntrinsicCall()
2297 Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0), in UpgradeIntrinsicCall()
2298 CI->getArgOperand(1),CI->getArgOperand(2), in UpgradeIntrinsicCall()
2301 auto *ResultTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2305 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0), in UpgradeIntrinsicCall()
2308 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2), in UpgradeIntrinsicCall()
2314 Rep = Builder.CreateCall(ELd, { Ptr, MaskVec, CI->getOperand(1) }); in UpgradeIntrinsicCall()
2316 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType()); in UpgradeIntrinsicCall()
2320 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0), in UpgradeIntrinsicCall()
2324 getX86MaskVec(Builder, CI->getArgOperand(2), in UpgradeIntrinsicCall()
2330 Rep = Builder.CreateCall(CSt, { CI->getArgOperand(1), Ptr, MaskVec }); in UpgradeIntrinsicCall()
2333 auto *ResultTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2335 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2), in UpgradeIntrinsicCall()
2342 Rep = Builder.CreateCall(Intr, { CI->getOperand(0), CI->getOperand(1), in UpgradeIntrinsicCall()
2356 if (CI->getNumArgOperands() == 3) { in UpgradeIntrinsicCall()
2357 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
2380 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned); in UpgradeIntrinsicCall()
2382 Value *Sel = CI->getArgOperand(2); in UpgradeIntrinsicCall()
2384 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel); in UpgradeIntrinsicCall()
2385 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel); in UpgradeIntrinsicCall()
2390 Rep = upgradeX86Rotate(Builder, *CI, false); in UpgradeIntrinsicCall()
2393 Rep = upgradeX86Rotate(Builder, *CI, true); in UpgradeIntrinsicCall()
2398 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask); in UpgradeIntrinsicCall()
2403 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask); in UpgradeIntrinsicCall()
2407 Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C)); in UpgradeIntrinsicCall()
2408 Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)}); in UpgradeIntrinsicCall()
2409 Rep = Builder.CreateZExt(Rep, CI->getType(), ""); in UpgradeIntrinsicCall()
2413 auto *VecTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2416 Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0), in UpgradeIntrinsicCall()
2430 auto *DstTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2439 Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask); in UpgradeIntrinsicCall()
2445 if (CI->getNumArgOperands() == 3) in UpgradeIntrinsicCall()
2446 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2447 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2452 Type *Ty = CI->getArgOperand(1)->getType(); in UpgradeIntrinsicCall()
2453 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty); in UpgradeIntrinsicCall()
2454 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2455 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2459 Type *EltTy = cast<VectorType>(CI->getType())->getElementType(); in UpgradeIntrinsicCall()
2462 Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0), in UpgradeIntrinsicCall()
2472 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
2473 Type *VT = CI->getType(); in UpgradeIntrinsicCall()
2488 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0), in UpgradeIntrinsicCall()
2489 CI->getArgOperand(1), ShuffleMask); in UpgradeIntrinsicCall()
2490 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, in UpgradeIntrinsicCall()
2491 CI->getArgOperand(3)); in UpgradeIntrinsicCall()
2495 cast<FixedVectorType>(CI->getArgOperand(0)->getType()) in UpgradeIntrinsicCall()
2498 cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
2504 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0), in UpgradeIntrinsicCall()
2505 CI->getArgOperand(0), in UpgradeIntrinsicCall()
2507 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2508 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2514 Value *Op = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2515 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount(); in UpgradeIntrinsicCall()
2521 if (CI->getNumArgOperands() == 3) in UpgradeIntrinsicCall()
2522 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2523 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2528 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat); in UpgradeIntrinsicCall()
2533 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat); in UpgradeIntrinsicCall()
2537 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat); in UpgradeIntrinsicCall()
2541 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat); in UpgradeIntrinsicCall()
2543 Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0), in UpgradeIntrinsicCall()
2544 CI->getArgOperand(1), in UpgradeIntrinsicCall()
2545 CI->getArgOperand(2), in UpgradeIntrinsicCall()
2546 CI->getArgOperand(3), in UpgradeIntrinsicCall()
2547 CI->getArgOperand(4), in UpgradeIntrinsicCall()
2550 Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0), in UpgradeIntrinsicCall()
2551 CI->getArgOperand(1), in UpgradeIntrinsicCall()
2552 CI->getArgOperand(2), in UpgradeIntrinsicCall()
2553 CI->getArgOperand(3), in UpgradeIntrinsicCall()
2554 CI->getArgOperand(4), in UpgradeIntrinsicCall()
2559 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
2560 Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), in UpgradeIntrinsicCall()
2565 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
2566 Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), in UpgradeIntrinsicCall()
2572 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
2573 Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift); in UpgradeIntrinsicCall()
2578 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
2579 Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift); in UpgradeIntrinsicCall()
2585 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2586 Value *Op1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
2587 unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
2588 auto *VecTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2599 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2600 Value *Op1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
2601 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
2603 cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
2639 if (CI->getNumArgOperands() == 5) in UpgradeIntrinsicCall()
2640 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, in UpgradeIntrinsicCall()
2641 CI->getArgOperand(3)); in UpgradeIntrinsicCall()
2645 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2646 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
2648 cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
2664 if (CI->getNumArgOperands() == 4) in UpgradeIntrinsicCall()
2665 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2666 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2671 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2672 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
2673 auto *VecTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2682 if (CI->getNumArgOperands() == 4) in UpgradeIntrinsicCall()
2683 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2684 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2695 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
2697 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
2702 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0); in UpgradeIntrinsicCall()
2703 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0); in UpgradeIntrinsicCall()
2706 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0; in UpgradeIntrinsicCall()
2707 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1; in UpgradeIntrinsicCall()
2725 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2726 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
2727 auto *VecTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2742 if (CI->getNumArgOperands() == 4) in UpgradeIntrinsicCall()
2743 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2744 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2747 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2748 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
2749 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
2761 if (CI->getNumArgOperands() == 4) in UpgradeIntrinsicCall()
2762 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2763 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2766 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2767 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
2768 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
2780 if (CI->getNumArgOperands() == 4) in UpgradeIntrinsicCall()
2781 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2782 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2784 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2785 Value *Op1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
2786 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
2787 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
2789 unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
2806 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, in UpgradeIntrinsicCall()
2807 CI->getArgOperand(3)); in UpgradeIntrinsicCall()
2811 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2812 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
2813 unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
2828 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2829 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2832 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2833 Value *Op1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
2834 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
2835 int NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
2844 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2845 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2848 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2849 Value *Op1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
2850 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
2851 int NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
2860 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2861 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2864 VectorType *FTy = cast<VectorType>(CI->getType()); in UpgradeIntrinsicCall()
2866 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy), in UpgradeIntrinsicCall()
2867 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); in UpgradeIntrinsicCall()
2869 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2870 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2873 VectorType *FTy = cast<VectorType>(CI->getType()); in UpgradeIntrinsicCall()
2875 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy)); in UpgradeIntrinsicCall()
2877 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); in UpgradeIntrinsicCall()
2879 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2880 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2883 VectorType *FTy = cast<VectorType>(CI->getType()); in UpgradeIntrinsicCall()
2885 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy), in UpgradeIntrinsicCall()
2886 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); in UpgradeIntrinsicCall()
2888 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2889 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2892 VectorType *FTy = cast<VectorType>(CI->getType()); in UpgradeIntrinsicCall()
2894 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy), in UpgradeIntrinsicCall()
2895 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); in UpgradeIntrinsicCall()
2897 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2898 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2900 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2901 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2902 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2904 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2905 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2906 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2908 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2909 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2910 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2920 { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
2921 CI->getArgOperand(4) }); in UpgradeIntrinsicCall()
2923 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2925 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2926 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2936 { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
2937 CI->getArgOperand(4) }); in UpgradeIntrinsicCall()
2939 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2941 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2942 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2952 { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
2953 CI->getArgOperand(4) }); in UpgradeIntrinsicCall()
2955 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2957 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2958 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2968 { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
2969 CI->getArgOperand(4) }); in UpgradeIntrinsicCall()
2971 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2973 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2974 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2987 { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
2988 CI->getArgOperand(4) }); in UpgradeIntrinsicCall()
2989 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2990 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2994 CI->getType()), in UpgradeIntrinsicCall()
2995 { CI->getArgOperand(0), Builder.getInt1(false) }); in UpgradeIntrinsicCall()
2996 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2997 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
3065 Rep = UpgradeX86MaskedShift(Builder, *CI, IID); in UpgradeIntrinsicCall()
3133 Rep = UpgradeX86MaskedShift(Builder, *CI, IID); in UpgradeIntrinsicCall()
3199 Rep = UpgradeX86MaskedShift(Builder, *CI, IID); in UpgradeIntrinsicCall()
3201 Rep = upgradeMaskedMove(Builder, *CI); in UpgradeIntrinsicCall()
3203 Rep = UpgradeMaskToInt(Builder, *CI); in UpgradeIntrinsicCall()
3209 Value *Ptr = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3213 Ptr, PointerType::getUnqual(CI->getType()), "cast"); in UpgradeIntrinsicCall()
3215 CI->getType(), BC, in UpgradeIntrinsicCall()
3216 Align(CI->getType()->getPrimitiveSizeInBits().getFixedSize() / 8)); in UpgradeIntrinsicCall()
3227 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3228 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
3243 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), in UpgradeIntrinsicCall()
3249 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, in UpgradeIntrinsicCall()
3252 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3253 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
3259 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), in UpgradeIntrinsicCall()
3264 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()), in UpgradeIntrinsicCall()
3278 Value *A = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3279 Value *B = CI->getArgOperand(1); in UpgradeIntrinsicCall()
3280 Value *C = CI->getArgOperand(2); in UpgradeIntrinsicCall()
3293 if (!isa<ConstantInt>(CI->getArgOperand(4)) || in UpgradeIntrinsicCall()
3294 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) { in UpgradeIntrinsicCall()
3295 Value *Ops[] = { A, B, C, CI->getArgOperand(4) }; in UpgradeIntrinsicCall()
3302 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID); in UpgradeIntrinsicCall()
3305 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), in UpgradeIntrinsicCall()
3317 PassThru = Builder.CreateExtractElement(CI->getArgOperand(2), in UpgradeIntrinsicCall()
3320 Rep = EmitX86ScalarSelect(Builder, CI->getArgOperand(3), in UpgradeIntrinsicCall()
3322 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), in UpgradeIntrinsicCall()
3338 Value *A = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3339 Value *B = CI->getArgOperand(1); in UpgradeIntrinsicCall()
3340 Value *C = CI->getArgOperand(2); in UpgradeIntrinsicCall()
3349 if (CI->getNumArgOperands() == 5 && in UpgradeIntrinsicCall()
3350 (!isa<ConstantInt>(CI->getArgOperand(4)) || in UpgradeIntrinsicCall()
3351 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) { in UpgradeIntrinsicCall()
3360 { A, B, C, CI->getArgOperand(4) }); in UpgradeIntrinsicCall()
3362 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), in UpgradeIntrinsicCall()
3368 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) : in UpgradeIntrinsicCall()
3369 IsMask3 ? CI->getArgOperand(2) : in UpgradeIntrinsicCall()
3370 CI->getArgOperand(0); in UpgradeIntrinsicCall()
3372 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in UpgradeIntrinsicCall()
3374 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in UpgradeIntrinsicCall()
3375 unsigned EltWidth = CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
3388 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3389 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
3402 if (CI->getNumArgOperands() == 5) { in UpgradeIntrinsicCall()
3410 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3411 CI->getArgOperand(2), CI->getArgOperand(4) }; in UpgradeIntrinsicCall()
3418 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
3420 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3421 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
3423 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma, in UpgradeIntrinsicCall()
3439 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) : in UpgradeIntrinsicCall()
3440 IsMask3 ? CI->getArgOperand(2) : in UpgradeIntrinsicCall()
3441 CI->getArgOperand(0); in UpgradeIntrinsicCall()
3443 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in UpgradeIntrinsicCall()
3447 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in UpgradeIntrinsicCall()
3448 unsigned EltWidth = CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
3465 Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1), in UpgradeIntrinsicCall()
3466 CI->getArgOperand(2), CI->getArgOperand(3) }; in UpgradeIntrinsicCall()
3467 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
3469 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) in UpgradeIntrinsicCall()
3470 : CI->getArgOperand(0); in UpgradeIntrinsicCall()
3471 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru); in UpgradeIntrinsicCall()
3476 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in UpgradeIntrinsicCall()
3493 Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1), in UpgradeIntrinsicCall()
3494 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
3495 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
3497 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) in UpgradeIntrinsicCall()
3498 : CI->getArgOperand(0); in UpgradeIntrinsicCall()
3499 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in UpgradeIntrinsicCall()
3505 Rep = UpgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm); in UpgradeIntrinsicCall()
3512 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in UpgradeIntrinsicCall()
3529 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3530 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
3531 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
3533 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) in UpgradeIntrinsicCall()
3534 : CI->getArgOperand(0); in UpgradeIntrinsicCall()
3535 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in UpgradeIntrinsicCall()
3542 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in UpgradeIntrinsicCall()
3559 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3560 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
3561 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
3563 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) in UpgradeIntrinsicCall()
3564 : CI->getArgOperand(0); in UpgradeIntrinsicCall()
3565 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in UpgradeIntrinsicCall()
3582 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3583 CI->getArgOperand(2)}; in UpgradeIntrinsicCall()
3585 Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
3591 Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(3), in UpgradeIntrinsicCall()
3597 CI->replaceAllUsesWith(CF); in UpgradeIntrinsicCall()
3600 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) { in UpgradeIntrinsicCall()
3603 Value *Arg = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3610 Value *Ptr = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3611 Value *Val = CI->getArgOperand(1); in UpgradeIntrinsicCall()
3616 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3617 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
3624 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3625 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
3632 Value *Arg = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3641 Value *Arg = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3651 CI->getArgOperand(0), "h2f"); in UpgradeIntrinsicCall()
3657 CI->replaceAllUsesWith(Rep); in UpgradeIntrinsicCall()
3658 CI->eraseFromParent(); in UpgradeIntrinsicCall()
3662 const auto &DefaultCase = [&NewFn, &CI]() -> void { in UpgradeIntrinsicCall()
3665 (CI->getCalledFunction()->getName() != NewFn->getName()) && in UpgradeIntrinsicCall()
3667 CI->setCalledFunction(NewFn); in UpgradeIntrinsicCall()
3689 SmallVector<Value *, 4> Args(CI->arg_operands().begin(), in UpgradeIntrinsicCall()
3690 CI->arg_operands().end()); in UpgradeIntrinsicCall()
3704 assert(CI->getNumArgOperands() == 3 && in UpgradeIntrinsicCall()
3707 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits(); in UpgradeIntrinsicCall()
3711 auto Iter = CI->arg_operands().begin(); in UpgradeIntrinsicCall()
3720 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
3725 assert(CI->getNumArgOperands() == 1 && in UpgradeIntrinsicCall()
3728 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()}); in UpgradeIntrinsicCall()
3732 Value *NullIsUnknownSize = CI->getNumArgOperands() == 2 in UpgradeIntrinsicCall()
3734 : CI->getArgOperand(2); in UpgradeIntrinsicCall()
3736 CI->getNumArgOperands() < 4 ? Builder.getFalse() : CI->getArgOperand(3); in UpgradeIntrinsicCall()
3738 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic}); in UpgradeIntrinsicCall()
3743 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
3747 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
3752 assert(CI->getNumArgOperands() == 4); in UpgradeIntrinsicCall()
3754 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1))) in UpgradeIntrinsicCall()
3758 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)}); in UpgradeIntrinsicCall()
3761 CI->eraseFromParent(); in UpgradeIntrinsicCall()
3766 assert(CI->getNumArgOperands() == 4 && in UpgradeIntrinsicCall()
3771 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), in UpgradeIntrinsicCall()
3772 CI->getArgOperand(3), Constant::getNullValue(Builder.getInt8PtrTy())}); in UpgradeIntrinsicCall()
3773 NewCall->takeName(CI); in UpgradeIntrinsicCall()
3774 CI->replaceAllUsesWith(NewCall); in UpgradeIntrinsicCall()
3775 CI->eraseFromParent(); in UpgradeIntrinsicCall()
3780 assert(CI->getNumArgOperands() == 4 && in UpgradeIntrinsicCall()
3785 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), in UpgradeIntrinsicCall()
3786 CI->getArgOperand(3), Constant::getNullValue(Builder.getInt8PtrTy())}); in UpgradeIntrinsicCall()
3787 CI->eraseFromParent(); in UpgradeIntrinsicCall()
3792 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)}); in UpgradeIntrinsicCall()
3799 SmallVector<Value *, 4> Args(CI->arg_operands().begin(), in UpgradeIntrinsicCall()
3800 CI->arg_operands().end()); in UpgradeIntrinsicCall()
3815 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3820 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
3834 if (CI->getNumOperands() == 0) in UpgradeIntrinsicCall()
3841 Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(0), in UpgradeIntrinsicCall()
3847 NewCall->takeName(CI); in UpgradeIntrinsicCall()
3848 CI->replaceAllUsesWith(TSC); in UpgradeIntrinsicCall()
3849 CI->eraseFromParent(); in UpgradeIntrinsicCall()
3861 SmallVector<Value *, 4> Args(CI->arg_operands().begin(), in UpgradeIntrinsicCall()
3862 CI->arg_operands().end()); in UpgradeIntrinsicCall()
3876 SmallVector<Value *, 4> Args(CI->arg_operands().begin(), in UpgradeIntrinsicCall()
3877 CI->arg_operands().end()); in UpgradeIntrinsicCall()
3885 NewCall->takeName(CI); in UpgradeIntrinsicCall()
3886 CI->replaceAllUsesWith(Res); in UpgradeIntrinsicCall()
3887 CI->eraseFromParent(); in UpgradeIntrinsicCall()
3898 SmallVector<Value *, 4> Args(CI->arg_operands().begin(), in UpgradeIntrinsicCall()
3899 CI->arg_operands().end()); in UpgradeIntrinsicCall()
3907 SmallVector<Value *, 4> Args(CI->arg_operands().begin(), in UpgradeIntrinsicCall()
3908 CI->arg_operands().end()); in UpgradeIntrinsicCall()
3910 NewCall->copyMetadata(*CI); in UpgradeIntrinsicCall()
3924 if (CI->getNumArgOperands() != 5) { in UpgradeIntrinsicCall()
3930 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3931 CI->getArgOperand(2), CI->getArgOperand(4)}; in UpgradeIntrinsicCall()
3935 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3)); in UpgradeIntrinsicCall()
3945 NewCall->takeName(CI); in UpgradeIntrinsicCall()
3946 CI->replaceAllUsesWith(NewCall); in UpgradeIntrinsicCall()
3947 CI->eraseFromParent(); in UpgradeIntrinsicCall()
3960 if (CallInst *CI = dyn_cast<CallInst>(U)) in UpgradeCallsToIntrinsic() local
3961 UpgradeIntrinsicCall(CI, NewFn); in UpgradeCallsToIntrinsic()
4097 CallInst *CI = dyn_cast<CallInst>(U); in UpgradeARCRuntime() local
4098 if (!CI || CI->getCalledFunction() != Fn) in UpgradeARCRuntime()
4101 IRBuilder<> Builder(CI->getParent(), CI->getIterator()); in UpgradeARCRuntime()
4107 if (NewFuncTy->getReturnType() != CI->getType() && in UpgradeARCRuntime()
4108 !CastInst::castIsValid(Instruction::BitCast, CI, in UpgradeARCRuntime()
4114 for (unsigned I = 0, E = CI->getNumArgOperands(); I != E; ++I) { in UpgradeARCRuntime()
4115 Value *Arg = CI->getArgOperand(I); in UpgradeARCRuntime()
4137 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind()); in UpgradeARCRuntime()
4138 NewCall->takeName(CI); in UpgradeARCRuntime()
4141 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType()); in UpgradeARCRuntime()
4143 if (!CI->use_empty()) in UpgradeARCRuntime()
4144 CI->replaceAllUsesWith(NewRetVal); in UpgradeARCRuntime()
4145 CI->eraseFromParent(); in UpgradeARCRuntime()