Lines Matching refs:CI
1270 static Value *UpgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, in UpgradeX86VPERMT2Intrinsics() argument
1272 Type *Ty = CI.getType(); in UpgradeX86VPERMT2Intrinsics()
1316 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1), in UpgradeX86VPERMT2Intrinsics()
1317 CI.getArgOperand(2) }; in UpgradeX86VPERMT2Intrinsics()
1323 Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID), in UpgradeX86VPERMT2Intrinsics()
1326 : Builder.CreateBitCast(CI.getArgOperand(1), in UpgradeX86VPERMT2Intrinsics()
1328 return EmitX86Select(Builder, CI.getArgOperand(3), V, PassThru); in UpgradeX86VPERMT2Intrinsics()
1331 static Value *UpgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, in UpgradeX86BinaryIntrinsics() argument
1333 Type *Ty = CI.getType(); in UpgradeX86BinaryIntrinsics()
1334 Value *Op0 = CI.getOperand(0); in UpgradeX86BinaryIntrinsics()
1335 Value *Op1 = CI.getOperand(1); in UpgradeX86BinaryIntrinsics()
1336 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in UpgradeX86BinaryIntrinsics()
1339 if (CI.arg_size() == 4) { // For masked intrinsics. in UpgradeX86BinaryIntrinsics()
1340 Value *VecSrc = CI.getOperand(2); in UpgradeX86BinaryIntrinsics()
1341 Value *Mask = CI.getOperand(3); in UpgradeX86BinaryIntrinsics()
1347 static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, in upgradeX86Rotate() argument
1349 Type *Ty = CI.getType(); in upgradeX86Rotate()
1350 Value *Src = CI.getArgOperand(0); in upgradeX86Rotate()
1351 Value *Amt = CI.getArgOperand(1); in upgradeX86Rotate()
1363 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in upgradeX86Rotate()
1366 if (CI.arg_size() == 4) { // For masked intrinsics. in upgradeX86Rotate()
1367 Value *VecSrc = CI.getOperand(2); in upgradeX86Rotate()
1368 Value *Mask = CI.getOperand(3); in upgradeX86Rotate()
1374 static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, in upgradeX86vpcom() argument
1376 Type *Ty = CI.getType(); in upgradeX86vpcom()
1377 Value *LHS = CI.getArgOperand(0); in upgradeX86vpcom()
1378 Value *RHS = CI.getArgOperand(1); in upgradeX86vpcom()
1413 static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, in upgradeX86ConcatShift() argument
1415 Type *Ty = CI.getType(); in upgradeX86ConcatShift()
1416 Value *Op0 = CI.getArgOperand(0); in upgradeX86ConcatShift()
1417 Value *Op1 = CI.getArgOperand(1); in upgradeX86ConcatShift()
1418 Value *Amt = CI.getArgOperand(2); in upgradeX86ConcatShift()
1433 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in upgradeX86ConcatShift()
1436 unsigned NumArgs = CI.arg_size(); in upgradeX86ConcatShift()
1438 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) : in upgradeX86ConcatShift()
1439 ZeroMask ? ConstantAggregateZero::get(CI.getType()) : in upgradeX86ConcatShift()
1440 CI.getArgOperand(0); in upgradeX86ConcatShift()
1441 Value *Mask = CI.getOperand(NumArgs - 1); in upgradeX86ConcatShift()
1492 static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) { in upgradeAbs() argument
1493 Type *Ty = CI.getType(); in upgradeAbs()
1494 Value *Op0 = CI.getArgOperand(0); in upgradeAbs()
1495 Function *F = Intrinsic::getDeclaration(CI.getModule(), Intrinsic::abs, Ty); in upgradeAbs()
1497 if (CI.arg_size() == 3) in upgradeAbs()
1498 Res = EmitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1)); in upgradeAbs()
1502 static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) { in upgradePMULDQ() argument
1503 Type *Ty = CI.getType(); in upgradePMULDQ()
1506 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty); in upgradePMULDQ()
1507 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty); in upgradePMULDQ()
1525 if (CI.arg_size() == 4) in upgradePMULDQ()
1526 Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2)); in upgradePMULDQ()
1554 static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, in upgradeMaskedCompare() argument
1556 Value *Op0 = CI.getArgOperand(0); in upgradeMaskedCompare()
1577 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1)); in upgradeMaskedCompare()
1580 Value *Mask = CI.getArgOperand(CI.arg_size() - 1); in upgradeMaskedCompare()
1586 static Value *UpgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, in UpgradeX86MaskedShift() argument
1588 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID); in UpgradeX86MaskedShift()
1590 { CI.getArgOperand(0), CI.getArgOperand(1) }); in UpgradeX86MaskedShift()
1591 return EmitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2)); in UpgradeX86MaskedShift()
1594 static Value* upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) { in upgradeMaskedMove() argument
1595 Value* A = CI.getArgOperand(0); in upgradeMaskedMove()
1596 Value* B = CI.getArgOperand(1); in upgradeMaskedMove()
1597 Value* Src = CI.getArgOperand(2); in upgradeMaskedMove()
1598 Value* Mask = CI.getArgOperand(3); in upgradeMaskedMove()
1609 static Value* UpgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) { in UpgradeMaskToInt() argument
1610 Value* Op = CI.getArgOperand(0); in UpgradeMaskToInt()
1611 Type* ReturnOp = CI.getType(); in UpgradeMaskToInt()
1612 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements(); in UpgradeMaskToInt()
1619 CallBase &CI, Value *&Rep) { in upgradeAVX512MaskToSelect() argument
1622 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits(); in upgradeAVX512MaskToSelect()
1623 unsigned EltWidth = CI.getType()->getScalarSizeInBits(); in upgradeAVX512MaskToSelect()
1763 bool IsFloat = CI.getType()->isFPOrFPVectorTy(); in upgradeAVX512MaskToSelect()
1845 SmallVector<Value *, 4> Args(CI.args()); in upgradeAVX512MaskToSelect()
1848 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID), in upgradeAVX512MaskToSelect()
1850 unsigned NumArgs = CI.arg_size(); in upgradeAVX512MaskToSelect()
1851 Rep = EmitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep, in upgradeAVX512MaskToSelect()
1852 CI.getArgOperand(NumArgs - 2)); in upgradeAVX512MaskToSelect()
1867 static Value *UpgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, in UpgradeARMIntrinsicCall() argument
1874 CI->getArgOperand(0), CI->getName()); in UpgradeARMIntrinsicCall()
1900 unsigned ID = CI->getIntrinsicID(); in UpgradeARMIntrinsicCall()
1906 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty}; in UpgradeARMIntrinsicCall()
1911 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(), in UpgradeARMIntrinsicCall()
1915 Tys = {CI->getType(), CI->getOperand(0)->getType(), in UpgradeARMIntrinsicCall()
1916 CI->getOperand(1)->getType(), V2I1Ty}; in UpgradeARMIntrinsicCall()
1919 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(), in UpgradeARMIntrinsicCall()
1920 CI->getOperand(2)->getType(), V2I1Ty}; in UpgradeARMIntrinsicCall()
1928 Tys = {CI->getOperand(1)->getType(), V2I1Ty}; in UpgradeARMIntrinsicCall()
1935 for (Value *Op : CI->args()) { in UpgradeARMIntrinsicCall()
1952 return Builder.CreateCall(Fn, Ops, CI->getName()); in UpgradeARMIntrinsicCall()
1959 void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) { in UpgradeIntrinsicCall() argument
1960 Function *F = CI->getCalledFunction(); in UpgradeIntrinsicCall()
1961 LLVMContext &C = CI->getContext(); in UpgradeIntrinsicCall()
1963 Builder.SetInsertPoint(CI->getParent(), CI->getIterator()); in UpgradeIntrinsicCall()
1991 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
1992 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
2006 CI->eraseFromParent(); in UpgradeIntrinsicCall()
2018 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2019 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
2031 CI->eraseFromParent(); in UpgradeIntrinsicCall()
2036 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2037 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
2048 CI->eraseFromParent(); in UpgradeIntrinsicCall()
2055 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2056 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
2064 CI->eraseFromParent(); in UpgradeIntrinsicCall()
2069 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1)); in UpgradeIntrinsicCall()
2070 UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
2074 CI->eraseFromParent(); in UpgradeIntrinsicCall()
2081 UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
2082 CI->getArgOperand(2), Aligned); in UpgradeIntrinsicCall()
2085 CI->eraseFromParent(); in UpgradeIntrinsicCall()
2096 CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2097 Rep = Builder.CreateSExt(Rep, CI->getType(), ""); in UpgradeIntrinsicCall()
2100 if (CI->getOperand(0)->getType()->isIntegerTy(8)) in UpgradeIntrinsicCall()
2102 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / in UpgradeIntrinsicCall()
2104 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy); in UpgradeIntrinsicCall()
2108 Value *Vec = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2119 CI->getType()), in UpgradeIntrinsicCall()
2120 {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
2122 if (CI->arg_size() == 4 && in UpgradeIntrinsicCall()
2123 (!isa<ConstantInt>(CI->getArgOperand(3)) || in UpgradeIntrinsicCall()
2124 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) { in UpgradeIntrinsicCall()
2128 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(3) }; in UpgradeIntrinsicCall()
2129 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), in UpgradeIntrinsicCall()
2134 CI->getType()), in UpgradeIntrinsicCall()
2135 {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
2137 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2138 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2141 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2142 Value *Op1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
2143 Value *Mask = CI->getArgOperand(2); in UpgradeIntrinsicCall()
2152 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType()) in UpgradeIntrinsicCall()
2154 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0)); in UpgradeIntrinsicCall()
2155 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2156 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2158 unsigned NumElts = CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
2159 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts); in UpgradeIntrinsicCall()
2160 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts); in UpgradeIntrinsicCall()
2175 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2177 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2178 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in UpgradeIntrinsicCall()
2180 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2182 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2183 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in UpgradeIntrinsicCall()
2186 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2188 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2189 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in UpgradeIntrinsicCall()
2191 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2193 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2194 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in UpgradeIntrinsicCall()
2196 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2198 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2199 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in UpgradeIntrinsicCall()
2202 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2204 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2206 Rep = Builder.CreateBitCast(Rep, CI->getType()); in UpgradeIntrinsicCall()
2209 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in UpgradeIntrinsicCall()
2210 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in UpgradeIntrinsicCall()
2225 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0), in UpgradeIntrinsicCall()
2227 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1), in UpgradeIntrinsicCall()
2238 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp, in UpgradeIntrinsicCall()
2243 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true); in UpgradeIntrinsicCall()
2245 Type *OpTy = CI->getArgOperand(0)->getType(); in UpgradeIntrinsicCall()
2256 { CI->getOperand(0), CI->getArgOperand(1) }); in UpgradeIntrinsicCall()
2257 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2259 Type *OpTy = CI->getArgOperand(0)->getType(); in UpgradeIntrinsicCall()
2279 { CI->getOperand(0), CI->getArgOperand(1) }); in UpgradeIntrinsicCall()
2280 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2282 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
2302 Value *Mask = Constant::getAllOnesValue(CI->getType()); in UpgradeIntrinsicCall()
2311 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
2312 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true); in UpgradeIntrinsicCall()
2314 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
2315 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false); in UpgradeIntrinsicCall()
2320 Value *Op = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2329 Rep = upgradeAbs(Builder, *CI); in UpgradeIntrinsicCall()
2335 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax); in UpgradeIntrinsicCall()
2341 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax); in UpgradeIntrinsicCall()
2347 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin); in UpgradeIntrinsicCall()
2353 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin); in UpgradeIntrinsicCall()
2358 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/false); in UpgradeIntrinsicCall()
2363 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/true); in UpgradeIntrinsicCall()
2369 CI->getArgOperand(1), in UpgradeIntrinsicCall()
2370 cast<VectorType>(CI->getType())->getElementType()); in UpgradeIntrinsicCall()
2371 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); in UpgradeIntrinsicCall()
2374 CI->getArgOperand(1), in UpgradeIntrinsicCall()
2375 cast<VectorType>(CI->getType())->getElementType()); in UpgradeIntrinsicCall()
2376 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); in UpgradeIntrinsicCall()
2378 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0); in UpgradeIntrinsicCall()
2380 Rep, cast<VectorType>(CI->getType())->getElementType()); in UpgradeIntrinsicCall()
2381 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); in UpgradeIntrinsicCall()
2400 auto *DstTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2401 Rep = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2414 else if (CI->arg_size() == 4 && in UpgradeIntrinsicCall()
2415 (!isa<ConstantInt>(CI->getArgOperand(3)) || in UpgradeIntrinsicCall()
2416 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) { in UpgradeIntrinsicCall()
2419 Function *F = Intrinsic::getDeclaration(CI->getModule(), IID, in UpgradeIntrinsicCall()
2421 Rep = Builder.CreateCall(F, { Rep, CI->getArgOperand(3) }); in UpgradeIntrinsicCall()
2427 if (CI->arg_size() >= 3) in UpgradeIntrinsicCall()
2428 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2429 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2432 auto *DstTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2433 Rep = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2443 if (CI->arg_size() >= 3) in UpgradeIntrinsicCall()
2444 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2445 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2450 UpgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
2451 CI->getArgOperand(2), Aligned); in UpgradeIntrinsicCall()
2453 auto *ResultTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2457 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0), in UpgradeIntrinsicCall()
2460 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2), in UpgradeIntrinsicCall()
2466 Rep = Builder.CreateCall(ELd, { Ptr, MaskVec, CI->getOperand(1) }); in UpgradeIntrinsicCall()
2468 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType()); in UpgradeIntrinsicCall()
2472 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0), in UpgradeIntrinsicCall()
2476 getX86MaskVec(Builder, CI->getArgOperand(2), in UpgradeIntrinsicCall()
2482 Rep = Builder.CreateCall(CSt, { CI->getArgOperand(1), Ptr, MaskVec }); in UpgradeIntrinsicCall()
2485 auto *ResultTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2487 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2), in UpgradeIntrinsicCall()
2494 Rep = Builder.CreateCall(Intr, { CI->getOperand(0), CI->getOperand(1), in UpgradeIntrinsicCall()
2508 if (CI->arg_size() == 3) { in UpgradeIntrinsicCall()
2509 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
2532 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned); in UpgradeIntrinsicCall()
2534 Value *Sel = CI->getArgOperand(2); in UpgradeIntrinsicCall()
2536 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel); in UpgradeIntrinsicCall()
2537 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel); in UpgradeIntrinsicCall()
2542 Rep = upgradeX86Rotate(Builder, *CI, false); in UpgradeIntrinsicCall()
2545 Rep = upgradeX86Rotate(Builder, *CI, true); in UpgradeIntrinsicCall()
2550 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask); in UpgradeIntrinsicCall()
2555 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask); in UpgradeIntrinsicCall()
2559 Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C)); in UpgradeIntrinsicCall()
2560 Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)}); in UpgradeIntrinsicCall()
2561 Rep = Builder.CreateZExt(Rep, CI->getType(), ""); in UpgradeIntrinsicCall()
2565 auto *VecTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2568 Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0), in UpgradeIntrinsicCall()
2582 auto *DstTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2591 Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask); in UpgradeIntrinsicCall()
2597 if (CI->arg_size() == 3) in UpgradeIntrinsicCall()
2598 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2599 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2604 Type *Ty = CI->getArgOperand(1)->getType(); in UpgradeIntrinsicCall()
2605 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty); in UpgradeIntrinsicCall()
2606 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2607 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2611 Type *EltTy = cast<VectorType>(CI->getType())->getElementType(); in UpgradeIntrinsicCall()
2614 Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0), in UpgradeIntrinsicCall()
2624 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
2625 Type *VT = CI->getType(); in UpgradeIntrinsicCall()
2640 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0), in UpgradeIntrinsicCall()
2641 CI->getArgOperand(1), ShuffleMask); in UpgradeIntrinsicCall()
2642 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, in UpgradeIntrinsicCall()
2643 CI->getArgOperand(3)); in UpgradeIntrinsicCall()
2647 cast<FixedVectorType>(CI->getArgOperand(0)->getType()) in UpgradeIntrinsicCall()
2650 cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
2656 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0), in UpgradeIntrinsicCall()
2657 CI->getArgOperand(0), in UpgradeIntrinsicCall()
2659 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2660 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2666 Value *Op = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2667 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount(); in UpgradeIntrinsicCall()
2673 if (CI->arg_size() == 3) in UpgradeIntrinsicCall()
2674 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2675 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2680 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat); in UpgradeIntrinsicCall()
2685 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat); in UpgradeIntrinsicCall()
2689 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat); in UpgradeIntrinsicCall()
2693 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat); in UpgradeIntrinsicCall()
2695 Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0), in UpgradeIntrinsicCall()
2696 CI->getArgOperand(1), in UpgradeIntrinsicCall()
2697 CI->getArgOperand(2), in UpgradeIntrinsicCall()
2698 CI->getArgOperand(3), in UpgradeIntrinsicCall()
2699 CI->getArgOperand(4), in UpgradeIntrinsicCall()
2702 Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0), in UpgradeIntrinsicCall()
2703 CI->getArgOperand(1), in UpgradeIntrinsicCall()
2704 CI->getArgOperand(2), in UpgradeIntrinsicCall()
2705 CI->getArgOperand(3), in UpgradeIntrinsicCall()
2706 CI->getArgOperand(4), in UpgradeIntrinsicCall()
2711 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
2712 Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), in UpgradeIntrinsicCall()
2717 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
2718 Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), in UpgradeIntrinsicCall()
2724 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
2725 Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift); in UpgradeIntrinsicCall()
2730 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
2731 Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift); in UpgradeIntrinsicCall()
2737 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2738 Value *Op1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
2739 unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
2740 auto *VecTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2751 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2752 Value *Op1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
2753 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
2755 cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
2791 if (CI->arg_size() == 5) in UpgradeIntrinsicCall()
2792 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, in UpgradeIntrinsicCall()
2793 CI->getArgOperand(3)); in UpgradeIntrinsicCall()
2797 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2798 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
2800 cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
2816 if (CI->arg_size() == 4) in UpgradeIntrinsicCall()
2817 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2818 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2823 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2824 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
2825 auto *VecTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2834 if (CI->arg_size() == 4) in UpgradeIntrinsicCall()
2835 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2836 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2847 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
2849 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
2854 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0); in UpgradeIntrinsicCall()
2855 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0); in UpgradeIntrinsicCall()
2858 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0; in UpgradeIntrinsicCall()
2859 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1; in UpgradeIntrinsicCall()
2877 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2878 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
2879 auto *VecTy = cast<FixedVectorType>(CI->getType()); in UpgradeIntrinsicCall()
2894 if (CI->arg_size() == 4) in UpgradeIntrinsicCall()
2895 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2896 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2899 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2900 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
2901 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
2913 if (CI->arg_size() == 4) in UpgradeIntrinsicCall()
2914 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2915 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2918 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2919 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
2920 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
2932 if (CI->arg_size() == 4) in UpgradeIntrinsicCall()
2933 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2934 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
2936 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2937 Value *Op1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
2938 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in UpgradeIntrinsicCall()
2939 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
2941 unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
2958 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, in UpgradeIntrinsicCall()
2959 CI->getArgOperand(3)); in UpgradeIntrinsicCall()
2963 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2964 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
2965 unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
2980 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
2981 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
2984 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
2985 Value *Op1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
2986 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
2987 int NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
2996 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
2997 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3000 Value *Op0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3001 Value *Op1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
3002 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
3003 int NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
3012 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3013 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3016 VectorType *FTy = cast<VectorType>(CI->getType()); in UpgradeIntrinsicCall()
3018 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy), in UpgradeIntrinsicCall()
3019 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); in UpgradeIntrinsicCall()
3021 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3022 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3025 VectorType *FTy = cast<VectorType>(CI->getType()); in UpgradeIntrinsicCall()
3027 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy)); in UpgradeIntrinsicCall()
3029 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); in UpgradeIntrinsicCall()
3031 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3032 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3035 VectorType *FTy = cast<VectorType>(CI->getType()); in UpgradeIntrinsicCall()
3037 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy), in UpgradeIntrinsicCall()
3038 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); in UpgradeIntrinsicCall()
3040 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3041 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3044 VectorType *FTy = cast<VectorType>(CI->getType()); in UpgradeIntrinsicCall()
3046 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy), in UpgradeIntrinsicCall()
3047 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); in UpgradeIntrinsicCall()
3049 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3050 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3052 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
3053 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3054 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3056 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
3057 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3058 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3060 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
3061 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3062 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3072 { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3073 CI->getArgOperand(4) }); in UpgradeIntrinsicCall()
3075 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
3077 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3078 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3088 { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3089 CI->getArgOperand(4) }); in UpgradeIntrinsicCall()
3091 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
3093 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3094 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3104 { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3105 CI->getArgOperand(4) }); in UpgradeIntrinsicCall()
3107 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
3109 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3110 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3120 { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3121 CI->getArgOperand(4) }); in UpgradeIntrinsicCall()
3123 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1)); in UpgradeIntrinsicCall()
3125 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3126 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3139 { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3140 CI->getArgOperand(4) }); in UpgradeIntrinsicCall()
3141 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, in UpgradeIntrinsicCall()
3142 CI->getArgOperand(2)); in UpgradeIntrinsicCall()
3146 CI->getType()), in UpgradeIntrinsicCall()
3147 { CI->getArgOperand(0), Builder.getInt1(false) }); in UpgradeIntrinsicCall()
3148 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, in UpgradeIntrinsicCall()
3149 CI->getArgOperand(1)); in UpgradeIntrinsicCall()
3217 Rep = UpgradeX86MaskedShift(Builder, *CI, IID); in UpgradeIntrinsicCall()
3285 Rep = UpgradeX86MaskedShift(Builder, *CI, IID); in UpgradeIntrinsicCall()
3351 Rep = UpgradeX86MaskedShift(Builder, *CI, IID); in UpgradeIntrinsicCall()
3353 Rep = upgradeMaskedMove(Builder, *CI); in UpgradeIntrinsicCall()
3355 Rep = UpgradeMaskToInt(Builder, *CI); in UpgradeIntrinsicCall()
3361 Value *Ptr = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3365 Ptr, PointerType::getUnqual(CI->getType()), "cast"); in UpgradeIntrinsicCall()
3367 CI->getType(), BC, in UpgradeIntrinsicCall()
3368 Align(CI->getType()->getPrimitiveSizeInBits().getFixedSize() / 8)); in UpgradeIntrinsicCall()
3379 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3380 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
3395 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), in UpgradeIntrinsicCall()
3401 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, in UpgradeIntrinsicCall()
3404 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3405 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
3411 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), in UpgradeIntrinsicCall()
3416 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()), in UpgradeIntrinsicCall()
3430 Value *A = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3431 Value *B = CI->getArgOperand(1); in UpgradeIntrinsicCall()
3432 Value *C = CI->getArgOperand(2); in UpgradeIntrinsicCall()
3445 if (!isa<ConstantInt>(CI->getArgOperand(4)) || in UpgradeIntrinsicCall()
3446 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) { in UpgradeIntrinsicCall()
3447 Value *Ops[] = { A, B, C, CI->getArgOperand(4) }; in UpgradeIntrinsicCall()
3454 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID); in UpgradeIntrinsicCall()
3457 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), in UpgradeIntrinsicCall()
3469 PassThru = Builder.CreateExtractElement(CI->getArgOperand(2), in UpgradeIntrinsicCall()
3472 Rep = EmitX86ScalarSelect(Builder, CI->getArgOperand(3), in UpgradeIntrinsicCall()
3474 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), in UpgradeIntrinsicCall()
3490 Value *A = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3491 Value *B = CI->getArgOperand(1); in UpgradeIntrinsicCall()
3492 Value *C = CI->getArgOperand(2); in UpgradeIntrinsicCall()
3501 if (CI->arg_size() == 5 && in UpgradeIntrinsicCall()
3502 (!isa<ConstantInt>(CI->getArgOperand(4)) || in UpgradeIntrinsicCall()
3503 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) { in UpgradeIntrinsicCall()
3512 { A, B, C, CI->getArgOperand(4) }); in UpgradeIntrinsicCall()
3514 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), in UpgradeIntrinsicCall()
3520 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) : in UpgradeIntrinsicCall()
3521 IsMask3 ? CI->getArgOperand(2) : in UpgradeIntrinsicCall()
3522 CI->getArgOperand(0); in UpgradeIntrinsicCall()
3524 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in UpgradeIntrinsicCall()
3526 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in UpgradeIntrinsicCall()
3527 unsigned EltWidth = CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
3540 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3541 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
3554 if (CI->arg_size() == 5) { in UpgradeIntrinsicCall()
3562 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3563 CI->getArgOperand(2), CI->getArgOperand(4) }; in UpgradeIntrinsicCall()
3570 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
3572 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3573 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
3575 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma, in UpgradeIntrinsicCall()
3591 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) : in UpgradeIntrinsicCall()
3592 IsMask3 ? CI->getArgOperand(2) : in UpgradeIntrinsicCall()
3593 CI->getArgOperand(0); in UpgradeIntrinsicCall()
3595 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in UpgradeIntrinsicCall()
3599 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in UpgradeIntrinsicCall()
3600 unsigned EltWidth = CI->getType()->getScalarSizeInBits(); in UpgradeIntrinsicCall()
3617 Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1), in UpgradeIntrinsicCall()
3618 CI->getArgOperand(2), CI->getArgOperand(3) }; in UpgradeIntrinsicCall()
3619 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
3621 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) in UpgradeIntrinsicCall()
3622 : CI->getArgOperand(0); in UpgradeIntrinsicCall()
3623 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru); in UpgradeIntrinsicCall()
3628 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in UpgradeIntrinsicCall()
3645 Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1), in UpgradeIntrinsicCall()
3646 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
3647 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
3649 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) in UpgradeIntrinsicCall()
3650 : CI->getArgOperand(0); in UpgradeIntrinsicCall()
3651 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in UpgradeIntrinsicCall()
3657 Rep = UpgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm); in UpgradeIntrinsicCall()
3664 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in UpgradeIntrinsicCall()
3681 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3682 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
3683 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
3685 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) in UpgradeIntrinsicCall()
3686 : CI->getArgOperand(0); in UpgradeIntrinsicCall()
3687 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in UpgradeIntrinsicCall()
3694 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in UpgradeIntrinsicCall()
3711 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3712 CI->getArgOperand(2) }; in UpgradeIntrinsicCall()
3713 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
3715 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) in UpgradeIntrinsicCall()
3716 : CI->getArgOperand(0); in UpgradeIntrinsicCall()
3717 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in UpgradeIntrinsicCall()
3734 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
3735 CI->getArgOperand(2)}; in UpgradeIntrinsicCall()
3737 Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
3743 Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(3), in UpgradeIntrinsicCall()
3749 CI->replaceAllUsesWith(CF); in UpgradeIntrinsicCall()
3752 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) { in UpgradeIntrinsicCall()
3755 Value *Arg = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3762 Value *Ptr = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3763 Value *Val = CI->getArgOperand(1); in UpgradeIntrinsicCall()
3768 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3769 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
3776 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3777 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
3784 Value *Arg = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3793 Value *Arg = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3803 CI->getArgOperand(0), "h2f"); in UpgradeIntrinsicCall()
3805 Rep = UpgradeARMIntrinsicCall(Name, CI, F, Builder); in UpgradeIntrinsicCall()
3811 CI->replaceAllUsesWith(Rep); in UpgradeIntrinsicCall()
3812 CI->eraseFromParent(); in UpgradeIntrinsicCall()
3817 if (CI->getFunctionType() == NewFn->getFunctionType()) { in UpgradeIntrinsicCall()
3820 (CI->getCalledFunction()->getName() != NewFn->getName()) && in UpgradeIntrinsicCall()
3822 CI->setCalledFunction(NewFn); in UpgradeIntrinsicCall()
3827 auto *OldST = cast<StructType>(CI->getType()); in UpgradeIntrinsicCall()
3833 SmallVector<Value *> Args(CI->args()); in UpgradeIntrinsicCall()
3840 CI->replaceAllUsesWith(Res); in UpgradeIntrinsicCall()
3841 CI->eraseFromParent(); in UpgradeIntrinsicCall()
3857 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
3871 assert(CI->arg_size() == 3 && in UpgradeIntrinsicCall()
3874 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits(); in UpgradeIntrinsicCall()
3878 auto Iter = CI->args().begin(); in UpgradeIntrinsicCall()
3887 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
3892 assert(CI->arg_size() == 1 && in UpgradeIntrinsicCall()
3895 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()}); in UpgradeIntrinsicCall()
3900 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2); in UpgradeIntrinsicCall()
3902 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3); in UpgradeIntrinsicCall()
3904 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic}); in UpgradeIntrinsicCall()
3909 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
3913 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
3918 assert(CI->arg_size() == 4); in UpgradeIntrinsicCall()
3920 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1))) in UpgradeIntrinsicCall()
3924 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)}); in UpgradeIntrinsicCall()
3927 CI->eraseFromParent(); in UpgradeIntrinsicCall()
3932 if (CI->arg_size() != 4) { in UpgradeIntrinsicCall()
3940 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), in UpgradeIntrinsicCall()
3941 CI->getArgOperand(3), Constant::getNullValue(Builder.getInt8PtrTy())}); in UpgradeIntrinsicCall()
3942 NewCall->takeName(CI); in UpgradeIntrinsicCall()
3943 CI->replaceAllUsesWith(NewCall); in UpgradeIntrinsicCall()
3944 CI->eraseFromParent(); in UpgradeIntrinsicCall()
3949 assert(CI->arg_size() == 4 && in UpgradeIntrinsicCall()
3954 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), in UpgradeIntrinsicCall()
3955 CI->getArgOperand(3), Constant::getNullValue(Builder.getInt8PtrTy())}); in UpgradeIntrinsicCall()
3956 CI->eraseFromParent(); in UpgradeIntrinsicCall()
3961 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)}); in UpgradeIntrinsicCall()
3968 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
3983 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
3988 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
4002 if (CI->getNumOperands() == 0) in UpgradeIntrinsicCall()
4009 Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(0), in UpgradeIntrinsicCall()
4015 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4016 CI->replaceAllUsesWith(TSC); in UpgradeIntrinsicCall()
4017 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4029 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4043 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4051 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4052 CI->replaceAllUsesWith(Res); in UpgradeIntrinsicCall()
4053 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4064 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4072 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4074 NewCall->copyMetadata(*CI); in UpgradeIntrinsicCall()
4088 if (CI->arg_size() != 5) { in UpgradeIntrinsicCall()
4094 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
4095 CI->getArgOperand(2), CI->getArgOperand(4)}; in UpgradeIntrinsicCall()
4097 AttributeList OldAttrs = CI->getAttributes(); in UpgradeIntrinsicCall()
4105 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3)); in UpgradeIntrinsicCall()
4115 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4116 CI->replaceAllUsesWith(NewCall); in UpgradeIntrinsicCall()
4117 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4267 CallInst *CI = dyn_cast<CallInst>(U); in UpgradeARCRuntime() local
4268 if (!CI || CI->getCalledFunction() != Fn) in UpgradeARCRuntime()
4271 IRBuilder<> Builder(CI->getParent(), CI->getIterator()); in UpgradeARCRuntime()
4277 if (NewFuncTy->getReturnType() != CI->getType() && in UpgradeARCRuntime()
4278 !CastInst::castIsValid(Instruction::BitCast, CI, in UpgradeARCRuntime()
4284 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) { in UpgradeARCRuntime()
4285 Value *Arg = CI->getArgOperand(I); in UpgradeARCRuntime()
4307 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind()); in UpgradeARCRuntime()
4308 NewCall->takeName(CI); in UpgradeARCRuntime()
4311 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType()); in UpgradeARCRuntime()
4313 if (!CI->use_empty()) in UpgradeARCRuntime()
4314 CI->replaceAllUsesWith(NewRetVal); in UpgradeARCRuntime()
4315 CI->eraseFromParent(); in UpgradeARCRuntime()