Lines Matching refs:Subtarget
109 : TargetLowering(TM), Subtarget(STI) { in X86TargetLowering()
110 bool UseX87 = !Subtarget.useSoftFloat() && Subtarget.hasX87(); in X86TargetLowering()
123 if (Subtarget.isAtom()) in X86TargetLowering()
125 else if (Subtarget.is64Bit()) in X86TargetLowering()
129 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo(); in X86TargetLowering()
134 if (Subtarget.hasSlowDivide32()) in X86TargetLowering()
136 if (Subtarget.hasSlowDivide64() && Subtarget.is64Bit()) in X86TargetLowering()
141 if (Subtarget.isTargetWindowsMSVC() || Subtarget.isTargetWindowsItanium()) { in X86TargetLowering()
160 if (Subtarget.getTargetTriple().isOSMSVCRT()) { in X86TargetLowering()
170 if (!Subtarget.canUseCMPXCHG8B()) in X86TargetLowering()
177 if (Subtarget.is64Bit()) in X86TargetLowering()
200 if (Subtarget.canUseCMOV()) { in X86TargetLowering()
203 if (Subtarget.is64Bit()) in X86TargetLowering()
211 if (Subtarget.is64Bit()) in X86TargetLowering()
217 LegalizeAction ShiftDoubleAction = Subtarget.isSHLDSlow() ? Custom : Legal; in X86TargetLowering()
222 if (Subtarget.is64Bit()) in X86TargetLowering()
226 if (!Subtarget.useSoftFloat()) { in X86TargetLowering()
290 if (!Subtarget.is64Bit()) { in X86TargetLowering()
296 if (Subtarget.hasSSE2()) { in X86TargetLowering()
303 if (Subtarget.is64Bit()) { in X86TargetLowering()
314 if (!Subtarget.hasSSE2()) { in X86TargetLowering()
317 if (Subtarget.is64Bit()) { in X86TargetLowering()
322 } else if (!Subtarget.is64Bit()) in X86TargetLowering()
351 if (Subtarget.is64Bit()) in X86TargetLowering()
362 if (!Subtarget.useSoftFloat() && Subtarget.hasX87()) { in X86TargetLowering()
372 if (Subtarget.hasBMI()) { in X86TargetLowering()
381 if (Subtarget.is64Bit()) { in X86TargetLowering()
387 if (Subtarget.hasLZCNT()) { in X86TargetLowering()
394 if (VT == MVT::i64 && !Subtarget.is64Bit()) in X86TargetLowering()
408 (!Subtarget.useSoftFloat() && Subtarget.hasF16C()) ? Custom : Expand); in X86TargetLowering()
428 if (Subtarget.is64Bit()) in X86TargetLowering()
430 if (Subtarget.hasPOPCNT()) { in X86TargetLowering()
439 if (Subtarget.is64Bit()) in X86TargetLowering()
447 if (!Subtarget.hasMOVBE()) in X86TargetLowering()
458 if (VT == MVT::i64 && !Subtarget.is64Bit()) in X86TargetLowering()
479 if (VT == MVT::i64 && !Subtarget.is64Bit()) in X86TargetLowering()
491 if (VT == MVT::i64 && !Subtarget.is64Bit()) in X86TargetLowering()
498 if (Subtarget.hasSSEPrefetch() || Subtarget.hasThreeDNow()) in X86TargetLowering()
514 if (!Subtarget.is64Bit()) in X86TargetLowering()
517 if (Subtarget.canUseCMPXCHG16B()) in X86TargetLowering()
521 if (!Subtarget.isTargetDarwin() && !Subtarget.isTargetELF() && in X86TargetLowering()
522 !Subtarget.isTargetCygMing() && !Subtarget.isTargetWin64() && in X86TargetLowering()
535 if (Subtarget.isTargetPS()) in X86TargetLowering()
543 bool Is64Bit = Subtarget.is64Bit(); in X86TargetLowering()
591 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) { in X86TargetLowering()
594 addRegisterClass(MVT::f16, Subtarget.hasAVX512() ? &X86::FR16XRegClass in X86TargetLowering()
596 addRegisterClass(MVT::f32, Subtarget.hasAVX512() ? &X86::FR32XRegClass in X86TargetLowering()
598 addRegisterClass(MVT::f64, Subtarget.hasAVX512() ? &X86::FR64XRegClass in X86TargetLowering()
673 } else if (!Subtarget.useSoftFloat() && Subtarget.hasSSE1() && in X86TargetLowering()
819 if (!Subtarget.useSoftFloat() && Subtarget.is64Bit() && Subtarget.hasSSE1()) { in X86TargetLowering()
820 addRegisterClass(MVT::f128, Subtarget.hasVLX() ? &X86::VR128XRegClass in X86TargetLowering()
972 if (!Subtarget.useSoftFloat() && Subtarget.hasMMX()) { in X86TargetLowering()
977 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE1()) { in X86TargetLowering()
978 addRegisterClass(MVT::v4f32, Subtarget.hasVLX() ? &X86::VR128XRegClass in X86TargetLowering()
1000 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) { in X86TargetLowering()
1001 addRegisterClass(MVT::v2f64, Subtarget.hasVLX() ? &X86::VR128XRegClass in X86TargetLowering()
1006 addRegisterClass(MVT::v16i8, Subtarget.hasVLX() ? &X86::VR128XRegClass in X86TargetLowering()
1008 addRegisterClass(MVT::v8i16, Subtarget.hasVLX() ? &X86::VR128XRegClass in X86TargetLowering()
1010 addRegisterClass(MVT::v8f16, Subtarget.hasVLX() ? &X86::VR128XRegClass in X86TargetLowering()
1012 addRegisterClass(MVT::v4i32, Subtarget.hasVLX() ? &X86::VR128XRegClass in X86TargetLowering()
1014 addRegisterClass(MVT::v2i64, Subtarget.hasVLX() ? &X86::VR128XRegClass in X86TargetLowering()
1099 if (VT == MVT::v2i64 && !Subtarget.is64Bit()) in X86TargetLowering()
1173 if (!Subtarget.hasAVX512()) in X86TargetLowering()
1209 if (!Subtarget.useSoftFloat() && Subtarget.hasSSSE3()) { in X86TargetLowering()
1226 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE41()) { in X86TargetLowering()
1282 if (Subtarget.is64Bit() && !Subtarget.hasAVX512()) { in X86TargetLowering()
1294 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE42()) { in X86TargetLowering()
1298 if (!Subtarget.useSoftFloat() && Subtarget.hasXOP()) { in X86TargetLowering()
1314 if (!Subtarget.useSoftFloat() && Subtarget.hasAVX()) { in X86TargetLowering()
1315 bool HasInt256 = Subtarget.hasInt256(); in X86TargetLowering()
1317 addRegisterClass(MVT::v32i8, Subtarget.hasVLX() ? &X86::VR256XRegClass in X86TargetLowering()
1319 addRegisterClass(MVT::v16i16, Subtarget.hasVLX() ? &X86::VR256XRegClass in X86TargetLowering()
1321 addRegisterClass(MVT::v16f16, Subtarget.hasVLX() ? &X86::VR256XRegClass in X86TargetLowering()
1323 addRegisterClass(MVT::v8i32, Subtarget.hasVLX() ? &X86::VR256XRegClass in X86TargetLowering()
1325 addRegisterClass(MVT::v8f32, Subtarget.hasVLX() ? &X86::VR256XRegClass in X86TargetLowering()
1327 addRegisterClass(MVT::v4i64, Subtarget.hasVLX() ? &X86::VR256XRegClass in X86TargetLowering()
1329 addRegisterClass(MVT::v4f64, Subtarget.hasVLX() ? &X86::VR256XRegClass in X86TargetLowering()
1382 if (!Subtarget.hasAVX512()) in X86TargetLowering()
1436 if (Subtarget.hasAnyFMA()) { in X86TargetLowering()
1517 setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom); in X86TargetLowering()
1560 if (!Subtarget.useSoftFloat() && !Subtarget.hasFP16() && in X86TargetLowering()
1561 Subtarget.hasF16C()) { in X86TargetLowering()
1582 if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) { in X86TargetLowering()
1607 if (!Subtarget.hasDQI()) { in X86TargetLowering()
1651 if (!Subtarget.useSoftFloat() && Subtarget.useAVX512Regs()) { in X86TargetLowering()
1652 bool HasBWI = Subtarget.hasBWI(); in X86TargetLowering()
1720 if (!Subtarget.hasVLX()) { in X86TargetLowering()
1821 setOperationAction(ISD::CTPOP, VT, Subtarget.hasBITALG() ? Legal : Custom); in X86TargetLowering()
1840 if (Subtarget.hasDQI()) { in X86TargetLowering()
1848 if (Subtarget.hasCDI()) { in X86TargetLowering()
1855 if (Subtarget.hasVPOPCNTDQ()) { in X86TargetLowering()
1905 if (Subtarget.hasVBMI2()) { in X86TargetLowering()
1923 if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) { in X86TargetLowering()
1931 if (Subtarget.hasDQI()) { in X86TargetLowering()
1965 if (Subtarget.hasDQI()) { in X86TargetLowering()
1976 if (Subtarget.hasCDI()) { in X86TargetLowering()
1982 if (Subtarget.hasVPOPCNTDQ()) { in X86TargetLowering()
1991 if (!Subtarget.useSoftFloat() && Subtarget.hasBWI()) { in X86TargetLowering()
2017 setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom); in X86TargetLowering()
2018 setOperationAction(ISD::MSTORE, VT, Subtarget.hasVLX() ? Legal : Custom); in X86TargetLowering()
2025 if (Subtarget.hasBITALG()) { in X86TargetLowering()
2031 if (!Subtarget.useSoftFloat() && Subtarget.hasFP16()) { in X86TargetLowering()
2092 if (Subtarget.useAVX512Regs()) { in X86TargetLowering()
2135 if (Subtarget.hasVLX()) { in X86TargetLowering()
2180 if (!Subtarget.useSoftFloat() && Subtarget.hasVLX()) { in X86TargetLowering()
2193 if (Subtarget.hasBWI()) { in X86TargetLowering()
2198 if (Subtarget.hasFP16()) { in X86TargetLowering()
2234 if (Subtarget.hasAMXTILE()) { in X86TargetLowering()
2242 if (!Subtarget.is64Bit()) { in X86TargetLowering()
2253 if (VT == MVT::i64 && !Subtarget.is64Bit()) in X86TargetLowering()
2271 if (!Subtarget.is64Bit()) { in X86TargetLowering()
2290 if (Subtarget.isTargetWin64()) { in X86TargetLowering()
2309 if (Subtarget.is32Bit() && in X86TargetLowering()
2310 (Subtarget.isTargetWindowsMSVC() || Subtarget.isTargetWindowsItanium())) in X86TargetLowering()
2376 computeRegisterProperties(Subtarget.getRegisterInfo()); in X86TargetLowering()
2396 PredictableSelectIsExpensive = Subtarget.getSchedModel().isOutOfOrder(); in X86TargetLowering()
2408 return Subtarget.isTargetMachO() && Subtarget.is64Bit(); in useLoadStackGuardNode()
2413 return Subtarget.getTargetTriple().isOSMSVCRT() && !Subtarget.isTargetMachO(); in useStackGuardXorFP()
2419 unsigned XorOp = Subtarget.is64Bit() ? X86::XOR64_FP : X86::XOR32_FP; in emitStackGuardXorFP()
2426 if ((VT == MVT::v32i1 || VT == MVT::v64i1) && Subtarget.hasAVX512() && in getPreferredVectorAction()
2427 !Subtarget.hasBWI()) in getPreferredVectorAction()
2431 !Subtarget.hasF16C() && VT.getVectorElementType() == MVT::f16) in getPreferredVectorAction()
2443 const X86Subtarget &Subtarget) { in handleMaskRegisterForCallingConv() argument
2458 if (NumElts == 32 && (!Subtarget.hasBWI() || CC != CallingConv::X86_RegCall)) in handleMaskRegisterForCallingConv()
2461 if (NumElts == 64 && Subtarget.hasBWI() && CC != CallingConv::X86_RegCall) { in handleMaskRegisterForCallingConv()
2462 if (Subtarget.useAVX512Regs()) in handleMaskRegisterForCallingConv()
2468 if (!isPowerOf2_32(NumElts) || (NumElts == 64 && !Subtarget.hasBWI()) || in handleMaskRegisterForCallingConv()
2479 if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512()) { in getRegisterTypeForCallingConv()
2485 handleMaskRegisterForCallingConv(NumElts, CC, Subtarget); in getRegisterTypeForCallingConv()
2495 if ((VT == MVT::f64 || VT == MVT::f80) && !Subtarget.is64Bit() && in getRegisterTypeForCallingConv()
2496 !Subtarget.hasX87()) in getRegisterTypeForCallingConv()
2510 if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512()) { in getNumRegistersForCallingConv()
2516 handleMaskRegisterForCallingConv(NumElts, CC, Subtarget); in getNumRegistersForCallingConv()
2527 if (!Subtarget.is64Bit() && !Subtarget.hasX87()) { in getNumRegistersForCallingConv()
2546 Subtarget.hasAVX512() && in getVectorTypeBreakdownForCallingConv()
2548 (VT.getVectorNumElements() == 64 && !Subtarget.hasBWI()) || in getVectorTypeBreakdownForCallingConv()
2557 if (VT == MVT::v64i1 && Subtarget.hasBWI() && !Subtarget.useAVX512Regs() && in getVectorTypeBreakdownForCallingConv()
2575 if (Subtarget.hasAVX512()) { in getSetCCResultType()
2585 if (LegalVT.getSimpleVT().isVector() && Subtarget.hasVLX()) { in getSetCCResultType()
2590 if (Subtarget.hasBWI() || EltVT.getSizeInBits() >= 32) in getSetCCResultType()
2629 if (Subtarget.is64Bit()) { in getByValTypeAlignment()
2638 if (Subtarget.hasSSE1()) in getByValTypeAlignment()
2651 (!Subtarget.isUnalignedMem16Slow() || Op.isAligned(Align(16)))) { in getOptimalMemOpType()
2653 if (Op.size() >= 64 && Subtarget.hasAVX512() && in getOptimalMemOpType()
2654 (Subtarget.getPreferVectorWidth() >= 512)) { in getOptimalMemOpType()
2655 return Subtarget.hasBWI() ? MVT::v64i8 : MVT::v16i32; in getOptimalMemOpType()
2658 if (Op.size() >= 32 && Subtarget.hasAVX() && in getOptimalMemOpType()
2659 (Subtarget.getPreferVectorWidth() >= 256)) { in getOptimalMemOpType()
2667 if (Subtarget.hasSSE2() && (Subtarget.getPreferVectorWidth() >= 128)) in getOptimalMemOpType()
2671 if (Subtarget.hasSSE1() && (Subtarget.is64Bit() || Subtarget.hasX87()) && in getOptimalMemOpType()
2672 (Subtarget.getPreferVectorWidth() >= 128)) in getOptimalMemOpType()
2675 Op.size() >= 8 && !Subtarget.is64Bit() && Subtarget.hasSSE2()) { in getOptimalMemOpType()
2688 if (Subtarget.is64Bit() && Op.size() >= 8) in getOptimalMemOpType()
2695 return Subtarget.hasSSE1(); in isSafeMemOpType()
2697 return Subtarget.hasSSE2(); in isSafeMemOpType()
2711 *Fast = !Subtarget.isUnalignedMem16Slow(); in allowsMisalignedMemoryAccesses()
2714 *Fast = !Subtarget.isUnalignedMem32Slow(); in allowsMisalignedMemoryAccesses()
2726 return (Alignment < 16 || !Subtarget.hasSSE41()); in allowsMisalignedMemoryAccesses()
2739 if (isPositionIndependent() && Subtarget.isPICStyleGOT()) in getJumpTableEncoding()
2781 return Subtarget.useSoftFloat(); in useSoftFloat()
2788 if (Subtarget.is64Bit()) in markLibCallAttributes()
2816 assert(isPositionIndependent() && Subtarget.isPICStyleGOT()); in LowerCustomJumpTableEntry()
2826 if (!Subtarget.is64Bit()) in getPICJumpTableRelocBase()
2840 if (Subtarget.isPICStyleRIPRel()) in getPICJumpTableRelocBaseExpr()
2856 RRC = Subtarget.is64Bit() ? &X86::GR64RegClass : &X86::GR32RegClass; in findRepresentativeClass()
2875 if (Subtarget.is64Bit()) in getAddressSpace()
2896 if (hasStackGuardSlotTLS(Subtarget.getTargetTriple())) { in getIRStackGuard()
2897 if (Subtarget.isTargetFuchsia()) { in getIRStackGuard()
2909 Offset = (Subtarget.is64Bit()) ? 0x28 : 0x14; in getIRStackGuard()
2922 Type *Ty = Subtarget.is64Bit() ? Type::getInt64Ty(M->getContext()) in getIRStackGuard()
2939 if (Subtarget.getTargetTriple().isWindowsMSVCEnvironment() || in insertSSPDeclarations()
2940 Subtarget.getTargetTriple().isWindowsItaniumEnvironment()) { in insertSSPDeclarations()
2960 hasStackGuardSlotTLS(Subtarget.getTargetTriple())) in insertSSPDeclarations()
2967 if (Subtarget.getTargetTriple().isWindowsMSVCEnvironment() || in getSDagStackGuard()
2968 Subtarget.getTargetTriple().isWindowsItaniumEnvironment()) { in getSDagStackGuard()
2976 if (Subtarget.getTargetTriple().isWindowsMSVCEnvironment() || in getSSPStackGuardCheck()
2977 Subtarget.getTargetTriple().isWindowsItaniumEnvironment()) { in getSSPStackGuardCheck()
2985 if (Subtarget.getTargetTriple().isOSContiki()) in getSafeStackPointerLocation()
2991 if (Subtarget.isTargetAndroid()) { in getSafeStackPointerLocation()
2994 int Offset = (Subtarget.is64Bit()) ? 0x48 : 0x24; in getSafeStackPointerLocation()
2999 if (Subtarget.isTargetFuchsia()) { in getSafeStackPointerLocation()
3060 CCValAssign &NextVA, const X86Subtarget &Subtarget) { in Passv64i1ArgInRegs() argument
3061 assert(Subtarget.hasBWI() && "Expected AVX512BW target!"); in Passv64i1ArgInRegs()
3062 assert(Subtarget.is32Bit() && "Expecting 32 bit target"); in Passv64i1ArgInRegs()
3136 if (!Subtarget.hasSSE1() && X86::FR32XRegClass.contains(VA.getLocReg())) { in LowerReturn()
3139 } else if (!Subtarget.hasSSE2() && in LowerReturn()
3163 if (Subtarget.is64Bit()) { in LowerReturn()
3171 if (!Subtarget.hasSSE2()) in LowerReturn()
3182 Subtarget); in LowerReturn()
3248 = (Subtarget.is64Bit() && !Subtarget.isTarget64BitILP32()) ? in LowerReturn()
3262 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo(); in LowerReturn()
3326 bool Darwin = Subtarget.getTargetTriple().isOSDarwin(); in getTypeForExtReturn()
3351 const SDLoc &Dl, const X86Subtarget &Subtarget, in getv64i1Argument() argument
3353 assert((Subtarget.hasBWI()) && "Expected AVX512BW target!"); in getv64i1Argument()
3354 assert(Subtarget.is32Bit() && "Expecting 32 bit target"); in getv64i1Argument()
3442 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); in LowerCallResult()
3465 if (!Subtarget.hasSSE1() && X86::FR32XRegClass.contains(VA.getLocReg())) { in LowerCallResult()
3471 } else if (!Subtarget.hasSSE2() && in LowerCallResult()
3486 if (!Subtarget.hasX87()) in LowerCallResult()
3497 getv64i1Argument(VA, RVLocs[++I], Chain, DAG, dl, Subtarget, &InFlag); in LowerCallResult()
3545 const X86Subtarget &Subtarget) { in hasCalleePopSRet() argument
3553 if (!Subtarget.is32Bit()) in hasCalleePopSRet()
3565 if (Subtarget.getTargetTriple().isOSMSVCRT()) in hasCalleePopSRet()
3569 if (Subtarget.isTargetMCU()) in hasCalleePopSRet()
3743 if (Subtarget.isTargetWindowsMSVC() && !Subtarget.is64Bit() && in LowerMemArgument()
3760 const X86Subtarget &Subtarget) { in get64BitArgumentGPRs() argument
3761 assert(Subtarget.is64Bit()); in get64BitArgumentGPRs()
3763 if (Subtarget.isCallingConvWin64(CallConv)) { in get64BitArgumentGPRs()
3779 const X86Subtarget &Subtarget) { in get64BitArgumentXMMs() argument
3780 assert(Subtarget.is64Bit()); in get64BitArgumentXMMs()
3781 if (Subtarget.isCallingConvWin64(CallConv)) { in get64BitArgumentXMMs()
3789 bool isSoftFloat = Subtarget.useSoftFloat(); in get64BitArgumentXMMs()
3790 if (isSoftFloat || !Subtarget.hasSSE1()) in get64BitArgumentXMMs()
3816 SelectionDAG &DAG, const X86Subtarget &Subtarget, in VarArgsLoweringHelper() argument
3818 : FuncInfo(FuncInfo), DL(Loc), DAG(DAG), Subtarget(Subtarget), in VarArgsLoweringHelper()
3822 FrameLowering(*Subtarget.getFrameLowering()), in VarArgsLoweringHelper()
3834 bool is64Bit() const { return Subtarget.is64Bit(); } in is64Bit()
3835 bool isWin64() const { return Subtarget.isCallingConvWin64(CallConv); } in isWin64()
3840 const X86Subtarget &Subtarget; member in __anonda2ec4e50511::VarArgsLoweringHelper
3866 ArrayRef<MCPhysReg> ArgGPRs = get64BitArgumentGPRs(CallConv, Subtarget); in createVarArgAreaAndStoreRegisters()
3868 get64BitArgumentXMMs(TheMachineFunction, CallConv, Subtarget); in createVarArgAreaAndStoreRegisters()
3872 assert(!(NumXMMRegs && !Subtarget.hasSSE1()) && in createVarArgAreaAndStoreRegisters()
3967 if (Subtarget.useAVX512Regs() && in forwardMustTailParameters()
3971 else if (Subtarget.hasAVX()) in forwardMustTailParameters()
3973 else if (Subtarget.hasSSE2()) in forwardMustTailParameters()
4026 if (F.hasExternalLinkage() && Subtarget.isTargetCygMing() && in LowerFormalArguments()
4031 bool Is64Bit = Subtarget.is64Bit(); in LowerFormalArguments()
4032 bool IsWin64 = Subtarget.isCallingConvWin64(CallConv); in LowerFormalArguments()
4075 getv64i1Argument(VA, ArgLocs[++I], Chain, DAG, dl, Subtarget); in LowerFormalArguments()
4087 RC = Subtarget.hasAVX512() ? &X86::FR16XRegClass : &X86::FR16RegClass; in LowerFormalArguments()
4089 RC = Subtarget.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass; in LowerFormalArguments()
4091 RC = Subtarget.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass; in LowerFormalArguments()
4099 RC = Subtarget.hasVLX() ? &X86::VR256XRegClass : &X86::VR256RegClass; in LowerFormalArguments()
4101 RC = Subtarget.hasVLX() ? &X86::VR128XRegClass : &X86::VR128RegClass; in LowerFormalArguments()
4163 if (Subtarget.is64Bit()) in LowerFormalArguments()
4204 VarArgsLoweringHelper(FuncInfo, dl, DAG, Subtarget, CallConv, CCInfo) in LowerFormalArguments()
4218 if (!canGuaranteeTCO(CallConv) && hasCalleePopSRet(Ins, Subtarget)) in LowerFormalArguments()
4270 if (Subtarget.isTargetWindowsMSVC() && !Subtarget.is64Bit() && in LowerMemOpCallTo()
4340 bool Is64Bit = Subtarget.is64Bit(); in LowerCall()
4341 bool IsWin64 = Subtarget.isCallingConvWin64(CallConv); in LowerCall()
4345 bool IsCalleePopSRet = !IsGuaranteeTCO && hasCalleePopSRet(Outs, Subtarget); in LowerCall()
4359 if (Subtarget.isPICStyleGOT() && !IsGuaranteeTCO && !IsMustTail) { in LowerCall()
4485 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo(); in LowerCall()
4556 Passv64i1ArgInRegs(dl, DAG, Arg, RegsToPass, VA, ArgLocs[++I], Subtarget); in LowerCall()
4588 if (Subtarget.isPICStyleGOT()) { in LowerCall()
4619 (Subtarget.hasSSE1() || !M->getModuleFlag("SkipRaxSetup"))) { in LowerCall()
4634 assert((Subtarget.hasSSE1() || !NumXMMRegs) in LowerCall()
4743 } else if (Subtarget.isTarget64BitILP32() && in LowerCall()
4809 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); in LowerCall()
4938 const Align StackAlignment = Subtarget.getFrameLowering()->getStackAlign(); in GetAlignedArgumentStackSize()
4939 const uint64_t SlotSize = Subtarget.getRegisterInfo()->getSlotSize(); in GetAlignedArgumentStackSize()
5062 bool IsCalleeWin64 = Subtarget.isCallingConvWin64(CalleeCC); in IsEligibleForTailCallOptimization()
5063 bool IsCallerWin64 = Subtarget.isCallingConvWin64(CallerCC); in IsEligibleForTailCallOptimization()
5084 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo(); in IsEligibleForTailCallOptimization()
5145 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo(); in IsEligibleForTailCallOptimization()
5175 const X86InstrInfo *TII = Subtarget.getInstrInfo(); in IsEligibleForTailCallOptimization()
5196 if (!Subtarget.is64Bit() && ((!isa<GlobalAddressSDNode>(Callee) && in IsEligibleForTailCallOptimization()
5225 X86::isCalleePop(CalleeCC, Subtarget.is64Bit(), isVarArg, in IsEligibleForTailCallOptimization()
5252 bool X86::mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget, in mayFoldLoad() argument
5261 if (!Subtarget.hasAVX() && !Subtarget.hasSSEUnalignedMem() && in mayFoldLoad()
5272 const X86Subtarget &Subtarget, in mayFoldLoadIntoBroadcastFromMem() argument
5274 assert(Subtarget.hasAVX() && "Expected AVX for broadcast from memory"); in mayFoldLoadIntoBroadcastFromMem()
5275 if (!X86::mayFoldLoad(Op, Subtarget, AssumeSingleUse)) in mayFoldLoadIntoBroadcastFromMem()
5359 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo(); in getReturnAddressFrameIndex()
5561 static bool useVPTERNLOG(const X86Subtarget &Subtarget, MVT VT) { in useVPTERNLOG() argument
5562 return Subtarget.hasVLX() || Subtarget.canExtendTo512DQ() || in useVPTERNLOG()
5739 return !IsFPSetCC || !Subtarget.isTarget64BitLP64() || !Subtarget.hasAVX(); in reduceSelectOfFPConstantLoads()
5745 if (VT.isVector() && Subtarget.hasAVX512()) in convertSelectOfConstantsToMath()
5775 (EltSizeInBits != 32 || !Subtarget.isPMULLDSlow())) in decomposeMulByConstant()
5826 return Subtarget.hasBMI(); in isCheapToSpeculateCttz()
5831 return Subtarget.hasLZCNT(); in isCheapToSpeculateCtlz()
5842 return !Subtarget.hasSSE2() || VT == MVT::f80; in ShouldShrinkFPConstant()
5846 return (VT == MVT::f64 && Subtarget.hasSSE2()) || in isScalarFPTypeInSSEReg()
5847 (VT == MVT::f32 && Subtarget.hasSSE1()) || VT == MVT::f16; in isScalarFPTypeInSSEReg()
5853 if (!Subtarget.hasAVX512() && !LoadVT.isVector() && BitcastVT.isVector() && in isLoadBitCastBeneficial()
5857 if (!Subtarget.hasDQI() && BitcastVT == MVT::v8i1 && LoadVT == MVT::i8) in isLoadBitCastBeneficial()
5875 unsigned MaxIntSize = Subtarget.is64Bit() ? 64 : 32; in canMergeStoresTo()
5880 if (MemVT.getSizeInBits() > Subtarget.getPreferVectorWidth()) in canMergeStoresTo()
5887 return Subtarget.hasFastLZCNT(); in isCtlzFast()
5901 if (!Subtarget.hasBMI()) in hasAndNotCompare()
5919 if (!Subtarget.hasSSE1() || VT.getSizeInBits() < 128) in hasAndNot()
5925 return Subtarget.hasSSE2(); in hasAndNot()
5949 if (Subtarget.hasAVX2()) in shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd()
5964 if ((Subtarget.hasFastVectorShiftMasks() && VT.isVector()) || in shouldFoldConstantShiftPairToMask()
5965 (Subtarget.hasFastScalarShiftMasks() && !VT.isVector())) { in shouldFoldConstantShiftPairToMask()
5982 if (VT == MVT::i64 && !Subtarget.is64Bit()) in shouldFoldMaskToVariableShiftPair()
5991 !Subtarget.isOSWindows()) in shouldExpandShift()
6329 static SDValue getZeroVector(MVT VT, const X86Subtarget &Subtarget, in getZeroVector() argument
6339 if (!Subtarget.hasSSE2() && VT.is128BitVector()) { in getZeroVector()
6344 assert((Subtarget.hasBWI() || VT.getVectorNumElements() <= 16) && in getZeroVector()
6461 const X86Subtarget &Subtarget, SelectionDAG &DAG, in widenSubVector() argument
6466 SDValue Res = ZeroNewElements ? getZeroVector(VT, Subtarget, DAG, dl) in widenSubVector()
6475 const X86Subtarget &Subtarget, SelectionDAG &DAG, in widenSubVector() argument
6483 return widenSubVector(VT, Vec, ZeroNewElements, Subtarget, DAG, dl); in widenSubVector()
6620 SDValue SplitOpsAndApply(SelectionDAG &DAG, const X86Subtarget &Subtarget, in SplitOpsAndApply() argument
6623 assert(Subtarget.hasSSE2() && "Target assumed to support at least SSE2"); in SplitOpsAndApply()
6625 if ((CheckBWI && Subtarget.useBWIRegs()) || in SplitOpsAndApply()
6626 (!CheckBWI && Subtarget.useAVX512Regs())) { in SplitOpsAndApply()
6631 } else if (Subtarget.hasAVX2()) { in SplitOpsAndApply()
6664 const X86Subtarget &Subtarget) { in getAVX512Node() argument
6665 assert(Subtarget.hasAVX512() && "AVX512 target expected"); in getAVX512Node()
6692 bool Widen = !(Subtarget.hasVLX() || VT.is512BitVector()); in getAVX512Node()
6714 Op = widenSubVector(Op, false, Subtarget, DAG, DL, 512); in getAVX512Node()
6727 const X86Subtarget &Subtarget) { in insert1BitVector() argument
6748 if ((!Subtarget.hasDQI() && NumElems == 8) || NumElems < 8) in insert1BitVector()
6749 WideOpVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1; in insert1BitVector()
6849 if (WideOpVT != MVT::v64i1 || Subtarget.is64Bit()) { in insert1BitVector()
7074 static SDValue getPack(SelectionDAG &DAG, const X86Subtarget &Subtarget, in getPack() argument
7079 bool UsePackUS = Subtarget.hasSSE41() || EltSizeInBits == 8; in getPack()
7143 const X86Subtarget &Subtarget, in getShuffleVectorZeroOrUndef() argument
7147 ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT); in getShuffleVectorZeroOrUndef()
8818 const X86Subtarget &Subtarget) { in LowerBuildVectorAsInsert() argument
8821 assert(((VT == MVT::v8i16 && Subtarget.hasSSE2()) || in LowerBuildVectorAsInsert()
8822 ((VT == MVT::v16i8 || VT == MVT::v4i32) && Subtarget.hasSSE41())) && in LowerBuildVectorAsInsert()
8840 V = getZeroVector(VT, Subtarget, DAG, dl); in LowerBuildVectorAsInsert()
8860 const X86Subtarget &Subtarget) { in LowerBuildVectorv16i8() argument
8861 if (NumNonZero > 8 && !Subtarget.hasSSE41()) in LowerBuildVectorv16i8()
8865 if (Subtarget.hasSSE41()) in LowerBuildVectorv16i8()
8867 Subtarget); in LowerBuildVectorv16i8()
8907 V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl); in LowerBuildVectorv16i8()
8926 const X86Subtarget &Subtarget) { in LowerBuildVectorv8i16() argument
8927 if (NumNonZero > 4 && !Subtarget.hasSSE41()) in LowerBuildVectorv8i16()
8932 Subtarget); in LowerBuildVectorv8i16()
8937 const X86Subtarget &Subtarget) { in LowerBuildVectorv4x32() argument
8942 if (Subtarget.hasSSE3() && !Subtarget.hasXOP() && in LowerBuildVectorv4x32()
9016 : getZeroVector(VT, Subtarget, DAG, SDLoc(Op)); in LowerBuildVectorv4x32()
9023 if (!Subtarget.hasSSE41()) in LowerBuildVectorv4x32()
9199 const X86Subtarget &Subtarget, in EltsFromConsecutiveLoads() argument
9336 VT.is256BitVector() && !Subtarget.hasInt256()) in EltsFromConsecutiveLoads()
9375 DAG, Subtarget, IsAfterLegalize); in EltsFromConsecutiveLoads()
9384 ((LoadSizeInBits == 16 && Subtarget.hasFP16()) || LoadSizeInBits == 32 || in EltsFromConsecutiveLoads()
9392 if (!Subtarget.hasSSE2() && VT == MVT::v4f32) in EltsFromConsecutiveLoads()
9409 if (ZeroMask.isZero() && isPowerOf2_32(NumElems) && Subtarget.hasAVX() && in EltsFromConsecutiveLoads()
9414 if (!Subtarget.hasAVX2() && ScalarSize < 32) in EltsFromConsecutiveLoads()
9452 RepeatVT, RepeatedLoads, DL, DAG, Subtarget, IsAfterLegalize)) { in EltsFromConsecutiveLoads()
9458 if (!Subtarget.hasAVX2() && in EltsFromConsecutiveLoads()
9461 Subtarget, in EltsFromConsecutiveLoads()
9481 const X86Subtarget &Subtarget, in combineToConsecutiveLoads() argument
9492 return EltsFromConsecutiveLoads(VT, Elts, DL, DAG, Subtarget, in combineToConsecutiveLoads()
9552 const X86Subtarget &Subtarget, in lowerBuildVectorAsBroadcast() argument
9557 if (!Subtarget.hasAVX()) in lowerBuildVectorAsBroadcast()
9583 if (!Sequence.empty() && Subtarget.hasCDI()) { in lowerBuildVectorAsBroadcast()
9603 if (!VT.is512BitVector() && !Subtarget.hasVLX()) { in lowerBuildVectorAsBroadcast()
9632 if (Subtarget.hasAVX()) { in lowerBuildVectorAsBroadcast()
9634 (SplatBitSize < 32 && Subtarget.hasAVX2())) { in lowerBuildVectorAsBroadcast()
9713 if (ConstSplatVal && (Subtarget.hasAVX2() || OptForSize)) { in lowerBuildVectorAsBroadcast()
9722 (ScalarSize == 64 && (IsGE256 || Subtarget.hasVLX())) || in lowerBuildVectorAsBroadcast()
9724 (OptForSize && (ScalarSize == 64 || Subtarget.hasAVX2()))) { in lowerBuildVectorAsBroadcast()
9748 if (!IsLoad && Subtarget.hasInt256() && in lowerBuildVectorAsBroadcast()
9761 (Subtarget.hasVLX() && ScalarSize == 64)) { in lowerBuildVectorAsBroadcast()
9774 if (Subtarget.hasInt256() && Ld.getValueType().isInteger() && in lowerBuildVectorAsBroadcast()
9786 if (ScalarSize == 16 && Subtarget.hasFP16() && IsGE256) in lowerBuildVectorAsBroadcast()
9902 const X86Subtarget &Subtarget) { in LowerBUILD_VECTORvXi1() argument
9947 if (VT == MVT::v64i1 && !Subtarget.is64Bit()) { in LowerBUILD_VECTORvXi1()
9968 if (VT == MVT::v64i1 && !Subtarget.is64Bit()) { in LowerBUILD_VECTORvXi1()
10183 const X86Subtarget &Subtarget, SelectionDAG &DAG, in isAddSubOrSubAdd() argument
10189 if (!Subtarget.hasSSE3() || !VT.isFloatingPoint()) in isAddSubOrSubAdd()
10309 static bool isFMAddSubOrFMSubAdd(const X86Subtarget &Subtarget, in isFMAddSubOrFMSubAdd() argument
10314 !Opnd0->hasNUsesOfValue(ExpectedUses, 0) || !Subtarget.hasAnyFMA()) in isFMAddSubOrFMSubAdd()
10338 const X86Subtarget &Subtarget, in lowerToAddSubOrFMAddSub() argument
10343 if (!isAddSubOrSubAdd(BV, Subtarget, DAG, Opnd0, Opnd1, NumExtracts, in lowerToAddSubOrFMAddSub()
10352 if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, NumExtracts)) { in lowerToAddSubOrFMAddSub()
10503 const X86Subtarget &Subtarget, in LowerToHorizontalOp() argument
10515 if (((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget.hasSSE3()) || in LowerToHorizontalOp()
10516 ((VT == MVT::v8i16 || VT == MVT::v4i32) && Subtarget.hasSSSE3()) || in LowerToHorizontalOp()
10517 ((VT == MVT::v8f32 || VT == MVT::v4f64) && Subtarget.hasAVX()) || in LowerToHorizontalOp()
10518 ((VT == MVT::v16i16 || VT == MVT::v8i32) && Subtarget.hasAVX2())) { in LowerToHorizontalOp()
10526 if (!Subtarget.hasAVX() || !VT.is256BitVector()) in LowerToHorizontalOp()
10617 static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
10627 const X86Subtarget &Subtarget, in lowerBuildVectorToBitOp() argument
10698 return LowerShift(Res, Subtarget, DAG); in lowerBuildVectorToBitOp()
10705 const X86Subtarget &Subtarget) { in materializeVectorConstant() argument
10716 if (Subtarget.hasSSE2() && ISD::isBuildVectorAllOnes(Op.getNode())) { in materializeVectorConstant()
10731 const X86Subtarget &Subtarget) { in createVariablePermute() argument
10746 IndicesVec = widenSubVector(IndicesVec, false, Subtarget, DAG, in createVariablePermute()
10763 Subtarget, DAG, SDLoc(IndicesVec)); in createVariablePermute()
10765 createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget); in createVariablePermute()
10771 SrcVec = widenSubVector(VT, SrcVec, false, Subtarget, DAG, SDLoc(SrcVec)); in createVariablePermute()
10805 if (Subtarget.hasSSSE3()) in createVariablePermute()
10809 if (Subtarget.hasVLX() && Subtarget.hasBWI()) in createVariablePermute()
10811 else if (Subtarget.hasSSSE3()) { in createVariablePermute()
10818 if (Subtarget.hasAVX()) { in createVariablePermute()
10821 } else if (Subtarget.hasSSSE3()) { in createVariablePermute()
10828 if (Subtarget.hasAVX()) { in createVariablePermute()
10833 } else if (Subtarget.hasSSE41()) { in createVariablePermute()
10837 getZeroVector(IndicesVT.getSimpleVT(), Subtarget, DAG, DL), in createVariablePermute()
10844 if (Subtarget.hasVLX() && Subtarget.hasVBMI()) in createVariablePermute()
10846 else if (Subtarget.hasXOP()) { in createVariablePermute()
10855 } else if (Subtarget.hasAVX()) { in createVariablePermute()
10873 return SplitOpsAndApply(DAG, Subtarget, DL, MVT::v32i8, Ops, in createVariablePermute()
10878 if (Subtarget.hasVLX() && Subtarget.hasBWI()) in createVariablePermute()
10880 else if (Subtarget.hasAVX()) { in createVariablePermute()
10886 DAG.getBitcast(MVT::v32i8, IndicesVec), DL, DAG, Subtarget)); in createVariablePermute()
10891 if (Subtarget.hasAVX2()) in createVariablePermute()
10893 else if (Subtarget.hasAVX()) { in createVariablePermute()
10899 if (Subtarget.hasXOP()) in createVariablePermute()
10915 if (Subtarget.hasAVX512()) { in createVariablePermute()
10916 if (!Subtarget.hasVLX()) { in createVariablePermute()
10918 SrcVec = widenSubVector(WidenSrcVT, SrcVec, false, Subtarget, DAG, in createVariablePermute()
10920 IndicesVec = widenSubVector(MVT::v8i64, IndicesVec, false, Subtarget, in createVariablePermute()
10923 DAG, Subtarget); in createVariablePermute()
10927 } else if (Subtarget.hasAVX()) { in createVariablePermute()
10935 if (Subtarget.hasXOP()) in createVariablePermute()
10950 if (Subtarget.hasVBMI()) in createVariablePermute()
10954 if (Subtarget.hasBWI()) in createVariablePermute()
10961 if (Subtarget.hasAVX512()) in createVariablePermute()
10999 const X86Subtarget &Subtarget) { in LowerBUILD_VECTORAsVariablePermute() argument
11039 return createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget); in LowerBUILD_VECTORAsVariablePermute()
11051 if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512()) in LowerBUILD_VECTOR()
11052 return LowerBUILD_VECTORvXi1(Op, DAG, Subtarget); in LowerBUILD_VECTOR()
11054 if (SDValue VectorConstant = materializeVectorConstant(Op, DAG, Subtarget)) in LowerBUILD_VECTOR()
11105 return widenSubVector(VT, NewBV, !UndefUpper, Subtarget, DAG, dl); in LowerBUILD_VECTOR()
11109 if (SDValue AddSub = lowerToAddSubOrFMAddSub(BV, Subtarget, DAG)) in LowerBUILD_VECTOR()
11111 if (SDValue HorizontalOp = LowerToHorizontalOp(BV, Subtarget, DAG)) in LowerBUILD_VECTOR()
11113 if (SDValue Broadcast = lowerBuildVectorAsBroadcast(BV, Subtarget, DAG)) in LowerBUILD_VECTOR()
11115 if (SDValue BitOp = lowerBuildVectorToBitOp(BV, Subtarget, DAG)) in LowerBUILD_VECTOR()
11172 assert(Subtarget.hasAVX() && "Must have AVX with >16-byte vector"); in LowerBUILD_VECTOR()
11195 EltVT == MVT::f64 || (EltVT == MVT::i64 && Subtarget.is64Bit()) || in LowerBUILD_VECTOR()
11196 (EltVT == MVT::i16 && Subtarget.hasFP16())) { in LowerBUILD_VECTOR()
11203 return getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG); in LowerBUILD_VECTOR()
11212 Item = getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG); in LowerBUILD_VECTOR()
11238 return getShuffleVectorZeroOrUndef(Item, Idx, NumZero > 0, Subtarget, DAG); in LowerBUILD_VECTOR()
11262 if (SDValue V = LowerBUILD_VECTORAsVariablePermute(Op, DAG, Subtarget)) in LowerBUILD_VECTOR()
11269 EltsFromConsecutiveLoads(VT, Ops, dl, DAG, Subtarget, false)) in LowerBUILD_VECTOR()
11276 if (Subtarget.hasAVX2() && EVTBits == 32 && Values.size() == 2) { in LowerBUILD_VECTOR()
11321 return getShuffleVectorZeroOrUndef(V2, Idx, true, Subtarget, DAG); in LowerBUILD_VECTOR()
11329 DAG, Subtarget)) in LowerBUILD_VECTOR()
11334 DAG, Subtarget)) in LowerBUILD_VECTOR()
11339 if (SDValue V = LowerBuildVectorv4x32(Op, DAG, Subtarget)) in LowerBUILD_VECTOR()
11348 Ops[i] = getZeroVector(VT, Subtarget, DAG, dl); in LowerBUILD_VECTOR()
11389 if (Subtarget.hasSSE41() && EltVT != MVT::f16) { in LowerBUILD_VECTOR()
11438 const X86Subtarget &Subtarget) { in LowerAVXCONCAT_VECTORS() argument
11474 SDValue Vec = NumZero ? getZeroVector(ResVT, Subtarget, DAG, dl) in LowerAVXCONCAT_VECTORS()
11496 const X86Subtarget &Subtarget, in LowerCONCAT_VECTORSvXi1() argument
11526 if ((!Subtarget.hasDQI() && NumElems == 8) || NumElems < 8) in LowerCONCAT_VECTORSvXi1()
11527 ShiftVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1; in LowerCONCAT_VECTORSvXi1()
11575 const X86Subtarget &Subtarget, in LowerCONCAT_VECTORS() argument
11579 return LowerCONCAT_VECTORSvXi1(Op, Subtarget, DAG); in LowerCONCAT_VECTORS()
11589 return LowerAVXCONCAT_VECTORS(Op, DAG, Subtarget); in LowerCONCAT_VECTORS()
12055 const X86Subtarget &Subtarget, in lowerShuffleWithPSHUFB() argument
12062 assert((Subtarget.hasSSSE3() && VT.is128BitVector()) || in lowerShuffleWithPSHUFB()
12063 (Subtarget.hasAVX2() && VT.is256BitVector()) || in lowerShuffleWithPSHUFB()
12064 (Subtarget.hasBWI() && VT.is512BitVector())); in lowerShuffleWithPSHUFB()
12106 const X86Subtarget &Subtarget, SelectionDAG &DAG,
12114 const X86Subtarget &Subtarget) { in lowerShuffleToEXPAND() argument
12127 Subtarget, DAG, DL); in lowerShuffleToEXPAND()
12128 SDValue ZeroVector = getZeroVector(VT, Subtarget, DAG, DL); in lowerShuffleToEXPAND()
12137 const X86Subtarget &Subtarget) { in matchShuffleWithUNPCK() argument
12175 if ((Subtarget.hasSSE41() || VT == MVT::v2i64 || VT == MVT::v2f64) && in matchShuffleWithUNPCK()
12194 V2 = Zero2 ? getZeroVector(VT, Subtarget, DAG, DL) : V1; in matchShuffleWithUNPCK()
12195 V1 = Zero1 ? getZeroVector(VT, Subtarget, DAG, DL) : V1; in matchShuffleWithUNPCK()
12277 const X86Subtarget &Subtarget) { in matchShuffleAsVTRUNC() argument
12278 if (!VT.is512BitVector() && !Subtarget.hasVLX()) in matchShuffleAsVTRUNC()
12287 if (SrcEltBits < 32 && !Subtarget.hasBWI()) in matchShuffleAsVTRUNC()
12314 const X86Subtarget &Subtarget, in getAVX512TruncNode() argument
12338 return widenSubVector(Trunc, ZeroUppers, Subtarget, DAG, DL, in getAVX512TruncNode()
12344 if (!Subtarget.hasVLX() && !SrcVT.is512BitVector()) { in getAVX512TruncNode()
12345 SDValue NewSrc = widenSubVector(Src, ZeroUppers, Subtarget, DAG, DL, 512); in getAVX512TruncNode()
12346 return getAVX512TruncNode(DL, DstVT, NewSrc, Subtarget, DAG, ZeroUppers); in getAVX512TruncNode()
12353 Trunc = widenSubVector(Trunc, ZeroUppers, Subtarget, DAG, DL, in getAVX512TruncNode()
12377 const X86Subtarget &Subtarget, in lowerShuffleWithVPMOV() argument
12380 if (!Subtarget.hasAVX512()) in lowerShuffleWithVPMOV()
12406 !Subtarget.hasBWI()) in lowerShuffleWithVPMOV()
12410 return getAVX512TruncNode(DL, VT, Src, Subtarget, DAG, !UndefUppers); in lowerShuffleWithVPMOV()
12420 const X86Subtarget &Subtarget, in lowerShuffleAsVTRUNC() argument
12424 if (!Subtarget.hasAVX512()) in lowerShuffleAsVTRUNC()
12433 if (SrcEltBits < 32 && !Subtarget.hasBWI()) in lowerShuffleAsVTRUNC()
12460 return getAVX512TruncNode(DL, VT, Src, Subtarget, DAG, !UndefUppers); in lowerShuffleAsVTRUNC()
12545 const X86Subtarget &Subtarget, in matchShuffleWithPACK() argument
12564 if (Subtarget.hasSSE41() || BitSize == 8) { in matchShuffleWithPACK()
12615 const X86Subtarget &Subtarget) { in lowerShuffleWithPACK() argument
12622 Subtarget, MaxStages)) in lowerShuffleWithPACK()
12629 if (NumStages != 1 && SizeBits == 128 && Subtarget.hasVLX()) in lowerShuffleWithPACK()
12636 (PackOpcode == X86ISD::PACKSS || Subtarget.hasSSE41())) in lowerShuffleWithPACK()
12665 const X86Subtarget &Subtarget, in lowerShuffleAsBitMask() argument
12671 if (EltVT == MVT::i64 && !Subtarget.is64Bit()) { in lowerShuffleAsBitMask()
12740 const X86Subtarget &Subtarget,
12809 const X86Subtarget &Subtarget, in lowerShuffleAsBlend() argument
12820 V1 = getZeroVector(VT, Subtarget, DAG, DL); in lowerShuffleAsBlend()
12822 V2 = getZeroVector(VT, Subtarget, DAG, DL); in lowerShuffleAsBlend()
12829 assert(Subtarget.hasAVX2() && "256-bit integer blends require AVX2!"); in lowerShuffleAsBlend()
12833 assert(Subtarget.hasAVX() && "256-bit float blends require AVX!"); in lowerShuffleAsBlend()
12840 assert(Subtarget.hasSSE41() && "128-bit blends require SSE41!"); in lowerShuffleAsBlend()
12844 assert(Subtarget.hasAVX2() && "v16i16 blends require AVX2!"); in lowerShuffleAsBlend()
12873 assert(Subtarget.hasAVX2() && "256-bit byte-blends require AVX2!"); in lowerShuffleAsBlend()
12876 assert(Subtarget.hasSSE41() && "128-bit byte-blends require SSE41!"); in lowerShuffleAsBlend()
12880 Subtarget, DAG)) in lowerShuffleAsBlend()
12883 if (Subtarget.hasBWI() && Subtarget.hasVLX()) { in lowerShuffleAsBlend()
12886 return getVectorMaskingNode(V2, MaskNode, V1, Subtarget, DAG); in lowerShuffleAsBlend()
12890 if (Subtarget.hasVLX()) in lowerShuffleAsBlend()
12947 Subtarget, DAG)) in lowerShuffleAsBlend()
12955 return getVectorMaskingNode(V2, MaskNode, V1, Subtarget, DAG); in lowerShuffleAsBlend()
13071 const X86Subtarget &Subtarget, SelectionDAG &DAG) { in lowerShuffleAsByteRotateAndPermute() argument
13072 if ((VT.is128BitVector() && !Subtarget.hasSSSE3()) || in lowerShuffleAsByteRotateAndPermute()
13073 (VT.is256BitVector() && !Subtarget.hasAVX2()) || in lowerShuffleAsByteRotateAndPermute()
13074 (VT.is512BitVector() && !Subtarget.hasBWI())) in lowerShuffleAsByteRotateAndPermute()
13170 const X86Subtarget &Subtarget, SelectionDAG &DAG) { in lowerShuffleAsDecomposedShuffleMerge() argument
13197 auto canonicalizeBroadcastableInput = [DL, VT, &Subtarget, in lowerShuffleAsDecomposedShuffleMerge()
13201 if (!Subtarget.hasAVX2() && (!Subtarget.hasAVX() || EltSizeInBits < 32 || in lowerShuffleAsDecomposedShuffleMerge()
13202 !X86::mayFoldLoad(Input, Subtarget))) in lowerShuffleAsDecomposedShuffleMerge()
13240 DL, VT, V1, V2, Mask, Subtarget, DAG)) in lowerShuffleAsDecomposedShuffleMerge()
13300 const X86Subtarget &Subtarget, in matchShuffleAsBitRotate() argument
13306 int MinSubElts = Subtarget.hasAVX512() ? std::max(32 / EltSizeInBits, 2) : 2; in matchShuffleAsBitRotate()
13325 const X86Subtarget &Subtarget, in lowerShuffleAsBitRotate() argument
13330 (VT.is128BitVector() && Subtarget.hasXOP()) || Subtarget.hasAVX512(); in lowerShuffleAsBitRotate()
13331 if (!IsLegal && Subtarget.hasSSE3()) in lowerShuffleAsBitRotate()
13336 Subtarget, Mask); in lowerShuffleAsBitRotate()
13476 const X86Subtarget &Subtarget, in lowerShuffleAsByteRotate() argument
13492 if (Subtarget.hasSSSE3()) { in lowerShuffleAsByteRotate()
13493 assert((!VT.is512BitVector() || Subtarget.hasBWI()) && in lowerShuffleAsByteRotate()
13533 const X86Subtarget &Subtarget, in lowerShuffleAsVALIGN() argument
13539 assert((Subtarget.hasVLX() || (!VT.is128BitVector() && !VT.is256BitVector())) in lowerShuffleAsVALIGN()
13555 const X86Subtarget &Subtarget, in lowerShuffleAsByteShiftMask() argument
13598 } else if (!Subtarget.hasSSSE3()) { in lowerShuffleAsByteShiftMask()
13642 const X86Subtarget &Subtarget) { in matchShuffleAsShift() argument
13687 unsigned MaxWidth = ((SizeInBits == 512) && !Subtarget.hasBWI() ? 64 : 128); in matchShuffleAsShift()
13704 const X86Subtarget &Subtarget, in lowerShuffleAsShift() argument
13715 Mask, 0, Zeroable, Subtarget); in lowerShuffleAsShift()
13720 Mask, Size, Zeroable, Subtarget); in lowerShuffleAsShift()
13888 ArrayRef<int> Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) { in lowerShuffleAsSpecificZeroOrAnyExtend() argument
13921 if (Subtarget.hasSSE41()) { in lowerShuffleAsSpecificZeroOrAnyExtend()
13962 if ((Scale * EltBits) == 64 && EltBits < 32 && Subtarget.hasSSE4A()) { in lowerShuffleAsSpecificZeroOrAnyExtend()
13987 if (Scale > 4 && EltBits == 8 && Subtarget.hasSSSE3()) { in lowerShuffleAsSpecificZeroOrAnyExtend()
14026 : getZeroVector(InputVT, Subtarget, DAG, DL); in lowerShuffleAsSpecificZeroOrAnyExtend()
14050 const APInt &Zeroable, const X86Subtarget &Subtarget, in lowerShuffleAsZeroOrAnyExtend() argument
14120 InputV, Mask, Subtarget, DAG); in lowerShuffleAsZeroOrAnyExtend()
14201 static bool isSoftFP16(T VT, const X86Subtarget &Subtarget) { in isSoftFP16() argument
14202 return VT.getScalarType() == MVT::f16 && !Subtarget.hasFP16(); in isSoftFP16()
14207 return ::isSoftFP16(VT, Subtarget); in isSoftFP16()
14216 const APInt &Zeroable, const X86Subtarget &Subtarget, in lowerShuffleAsElementInsertion() argument
14221 if (isSoftFP16(EltVT, Subtarget)) in lowerShuffleAsElementInsertion()
14244 if (EltVT == MVT::i8 || (EltVT == MVT::i16 && !Subtarget.hasFP16())) { in lowerShuffleAsElementInsertion()
14323 const X86Subtarget &Subtarget, in lowerShuffleAsTruncBroadcast() argument
14325 assert(Subtarget.hasAVX2() && in lowerShuffleAsTruncBroadcast()
14451 const X86Subtarget &Subtarget, in lowerShuffleAsBroadcast() argument
14453 if (!((Subtarget.hasSSE3() && VT == MVT::v2f64) || in lowerShuffleAsBroadcast()
14454 (Subtarget.hasAVX() && VT.isFloatingPoint()) || in lowerShuffleAsBroadcast()
14455 (Subtarget.hasAVX2() && VT.isInteger()))) in lowerShuffleAsBroadcast()
14461 unsigned Opcode = (VT == MVT::v2f64 && !Subtarget.hasAVX2()) in lowerShuffleAsBroadcast()
14464 bool BroadcastFromReg = (Opcode == X86ISD::MOVDDUP) || Subtarget.hasAVX2(); in lowerShuffleAsBroadcast()
14533 DL, VT, V, BroadcastIdx, Subtarget, DAG)) in lowerShuffleAsBroadcast()
14607 if (Subtarget.hasAVX()) { in lowerShuffleAsBroadcast()
14757 const X86Subtarget &Subtarget, SelectionDAG &DAG) { in lowerShuffleAsPermuteAndUnpack() argument
14872 const X86Subtarget &Subtarget, in lowerV2F64Shuffle() argument
14881 Mask, Subtarget, DAG)) in lowerV2F64Shuffle()
14888 if (Subtarget.hasAVX()) { in lowerV2F64Shuffle()
14906 if (Subtarget.hasAVX2()) in lowerV2F64Shuffle()
14913 DL, MVT::v2f64, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV2F64Shuffle()
14920 DL, MVT::v2f64, V2, V1, InverseMask, Zeroable, Subtarget, DAG)) in lowerV2F64Shuffle()
14934 if (Subtarget.hasSSE41()) in lowerV2F64Shuffle()
14936 Zeroable, Subtarget, DAG)) in lowerV2F64Shuffle()
14956 const X86Subtarget &Subtarget, in lowerV2I64Shuffle() argument
14965 Mask, Subtarget, DAG)) in lowerV2I64Shuffle()
14986 if (Subtarget.hasAVX2()) in lowerV2I64Shuffle()
14992 Zeroable, Subtarget, DAG)) in lowerV2I64Shuffle()
14998 DL, MVT::v2i64, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV2I64Shuffle()
15004 DL, MVT::v2i64, V2, V1, InverseMask, Zeroable, Subtarget, DAG)) in lowerV2I64Shuffle()
15009 bool IsBlendSupported = Subtarget.hasSSE41(); in lowerV2I64Shuffle()
15012 Zeroable, Subtarget, DAG)) in lowerV2I64Shuffle()
15021 if (Subtarget.hasSSSE3()) { in lowerV2I64Shuffle()
15022 if (Subtarget.hasVLX()) in lowerV2I64Shuffle()
15024 Subtarget, DAG)) in lowerV2I64Shuffle()
15028 Subtarget, DAG)) in lowerV2I64Shuffle()
15036 Subtarget, DAG); in lowerV2I64Shuffle()
15147 const X86Subtarget &Subtarget, in lowerV4F32Shuffle() argument
15158 Mask, Subtarget, DAG)) in lowerV4F32Shuffle()
15162 if (Subtarget.hasSSE3()) { in lowerV4F32Shuffle()
15169 if (Subtarget.hasAVX()) { in lowerV4F32Shuffle()
15178 if (!Subtarget.hasSSE2()) { in lowerV4F32Shuffle()
15191 if (Subtarget.hasAVX2()) in lowerV4F32Shuffle()
15202 DL, MVT::v4f32, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV4F32Shuffle()
15205 if (Subtarget.hasSSE41()) { in lowerV4F32Shuffle()
15207 Zeroable, Subtarget, DAG)) in lowerV4F32Shuffle()
15222 if (!Subtarget.hasSSE2()) { in lowerV4F32Shuffle()
15243 const X86Subtarget &Subtarget, in lowerV4I32Shuffle() argument
15253 Zeroable, Subtarget, DAG)) in lowerV4I32Shuffle()
15262 Mask, Subtarget, DAG)) in lowerV4I32Shuffle()
15282 if (Subtarget.hasAVX2()) in lowerV4I32Shuffle()
15288 Zeroable, Subtarget, DAG)) in lowerV4I32Shuffle()
15294 DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV4I32Shuffle()
15299 bool IsBlendSupported = Subtarget.hasSSE41(); in lowerV4I32Shuffle()
15302 Zeroable, Subtarget, DAG)) in lowerV4I32Shuffle()
15306 Zeroable, Subtarget, DAG)) in lowerV4I32Shuffle()
15315 if (Subtarget.hasSSSE3()) { in lowerV4I32Shuffle()
15316 if (Subtarget.hasVLX()) in lowerV4I32Shuffle()
15318 Subtarget, DAG)) in lowerV4I32Shuffle()
15322 Subtarget, DAG)) in lowerV4I32Shuffle()
15334 Subtarget, DAG); in lowerV4I32Shuffle()
15338 Mask, Subtarget, DAG)) in lowerV4I32Shuffle()
15371 const X86Subtarget &Subtarget, SelectionDAG &DAG) { in lowerV8I16GeneralSingleInputShuffle() argument
15605 return lowerV8I16GeneralSingleInputShuffle(DL, VT, V, Mask, Subtarget, DAG); in lowerV8I16GeneralSingleInputShuffle()
15928 const X86Subtarget &Subtarget, in lowerV8I16Shuffle() argument
15937 Zeroable, Subtarget, DAG)) in lowerV8I16Shuffle()
15942 Subtarget, DAG)) in lowerV8I16Shuffle()
15950 Zeroable, Subtarget, DAG)) in lowerV8I16Shuffle()
15955 Mask, Subtarget, DAG)) in lowerV8I16Shuffle()
15960 Subtarget, DAG)) in lowerV8I16Shuffle()
15969 Subtarget)) in lowerV8I16Shuffle()
15974 Subtarget, DAG)) in lowerV8I16Shuffle()
15980 Subtarget, DAG); in lowerV8I16Shuffle()
15989 Zeroable, Subtarget, DAG)) in lowerV8I16Shuffle()
15993 if (Subtarget.hasSSE4A()) in lowerV8I16Shuffle()
16001 DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV8I16Shuffle()
16006 bool IsBlendSupported = Subtarget.hasSSE41(); in lowerV8I16Shuffle()
16009 Zeroable, Subtarget, DAG)) in lowerV8I16Shuffle()
16013 Zeroable, Subtarget, DAG)) in lowerV8I16Shuffle()
16022 Subtarget)) in lowerV8I16Shuffle()
16027 Subtarget, DAG)) in lowerV8I16Shuffle()
16032 Subtarget, DAG)) in lowerV8I16Shuffle()
16041 Zeroable, Subtarget, DAG)) in lowerV8I16Shuffle()
16048 if ((NumEvenDrops == 1 || NumEvenDrops == 2) && Subtarget.hasSSE41() && in lowerV8I16Shuffle()
16049 !Subtarget.hasVLX()) { in lowerV8I16Shuffle()
16051 if (NumEvenDrops == 2 && Subtarget.hasAVX2() && in lowerV8I16Shuffle()
16056 getZeroVector(MVT::v16i16, Subtarget, DAG, DL), in lowerV8I16Shuffle()
16085 bool HasSSE41 = Subtarget.hasSSE41(); in lowerV8I16Shuffle()
16098 Mask, Subtarget, DAG)) in lowerV8I16Shuffle()
16103 if (!IsBlendSupported && Subtarget.hasSSSE3()) { in lowerV8I16Shuffle()
16112 Mask, Subtarget, DAG); in lowerV8I16Shuffle()
16118 const X86Subtarget &Subtarget, in lowerV8F16Shuffle() argument
16125 if (Subtarget.hasFP16()) { in lowerV8F16Shuffle()
16129 Mask, Subtarget, DAG)) in lowerV8F16Shuffle()
16134 DL, MVT::v8f16, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV8F16Shuffle()
16149 const X86Subtarget &Subtarget, in lowerShuffleWithPERMV() argument
16154 if (!VT.is512BitVector() && !Subtarget.hasVLX()) { in lowerShuffleWithPERMV()
16155 V1 = widenSubVector(V1, false, Subtarget, DAG, DL, 512); in lowerShuffleWithPERMV()
16156 V2 = widenSubVector(V2, false, Subtarget, DAG, DL, 512); in lowerShuffleWithPERMV()
16167 MaskNode = widenSubVector(MaskNode, false, Subtarget, DAG, DL, 512); in lowerShuffleWithPERMV()
16193 const X86Subtarget &Subtarget, in lowerV16I8Shuffle() argument
16201 Zeroable, Subtarget, DAG)) in lowerV16I8Shuffle()
16206 Subtarget, DAG)) in lowerV16I8Shuffle()
16211 Subtarget)) in lowerV16I8Shuffle()
16216 Zeroable, Subtarget, DAG)) in lowerV16I8Shuffle()
16221 Subtarget, DAG)) in lowerV16I8Shuffle()
16225 Subtarget, DAG)) in lowerV16I8Shuffle()
16229 if (Subtarget.hasSSE4A()) in lowerV16I8Shuffle()
16240 Mask, Subtarget, DAG)) in lowerV16I8Shuffle()
16245 Subtarget, DAG)) in lowerV16I8Shuffle()
16350 Zeroable, Subtarget, DAG)) in lowerV16I8Shuffle()
16359 Zeroable, Subtarget, DAG)) in lowerV16I8Shuffle()
16382 if (Subtarget.hasSSSE3() && (IsSingleInput || NumEvenDrops != 1)) { in lowerV16I8Shuffle()
16393 if (Subtarget.hasSSE41()) in lowerV16I8Shuffle()
16395 Zeroable, Subtarget, DAG)) in lowerV16I8Shuffle()
16407 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG)) in lowerV16I8Shuffle()
16411 if (Subtarget.hasVBMI()) in lowerV16I8Shuffle()
16412 return lowerShuffleWithPERMV(DL, MVT::v16i8, Mask, V1, V2, Subtarget, in lowerV16I8Shuffle()
16416 if (Subtarget.hasXOP()) { in lowerV16I8Shuffle()
16424 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG)) in lowerV16I8Shuffle()
16434 DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV16I8Shuffle()
16491 Subtarget, DAG); in lowerV16I8Shuffle()
16528 SDValue Zero = getZeroVector(MVT::v16i8, Subtarget, DAG, DL); in lowerV16I8Shuffle()
16549 const X86Subtarget &Subtarget, in lower128BitShuffle() argument
16553 return lowerV2I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower128BitShuffle()
16555 return lowerV2F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower128BitShuffle()
16557 return lowerV4I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower128BitShuffle()
16559 return lowerV4F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower128BitShuffle()
16561 return lowerV8I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower128BitShuffle()
16563 return lowerV8F16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower128BitShuffle()
16565 return lowerV16I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower128BitShuffle()
16681 const X86Subtarget &Subtarget, in lowerShuffleAsSplitOrBlend() argument
16707 return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Subtarget, in lowerShuffleAsSplitOrBlend()
16726 return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Subtarget, in lowerShuffleAsSplitOrBlend()
16770 SelectionDAG &DAG, const X86Subtarget &Subtarget) { in lowerShuffleAsLanePermuteAndPermute() argument
16774 bool CanUseSublanes = Subtarget.hasAVX2() && V2.isUndef(); in lowerShuffleAsLanePermuteAndPermute()
16858 if (!Subtarget.hasFastVariableCrossLaneShuffle()) in lowerShuffleAsLanePermuteAndPermute()
16873 SelectionDAG &DAG, const X86Subtarget &Subtarget) { in lowerShuffleAsLanePermuteAndShuffle() argument
16890 if (!Subtarget.hasAVX2()) { in lowerShuffleAsLanePermuteAndShuffle()
16937 const X86Subtarget &Subtarget, in lowerV2X128Shuffle() argument
16943 if ((SplatLo || SplatHi) && !Subtarget.hasAVX512() && V1.hasOneUse() && in lowerV2X128Shuffle()
16944 X86::mayFoldLoad(peekThroughOneUseBitcasts(V1), Subtarget)) { in lowerV2X128Shuffle()
16954 if (Subtarget.hasAVX2()) in lowerV2X128Shuffle()
16973 getZeroVector(VT, Subtarget, DAG, DL), LoV, in lowerV2X128Shuffle()
16983 Subtarget, DAG)) in lowerV2X128Shuffle()
17007 if (Subtarget.hasVLX()) { in lowerV2X128Shuffle()
17056 const X86Subtarget &Subtarget, SelectionDAG &DAG) { in lowerShuffleAsLanePermuteAndRepeatedMask() argument
17316 const X86Subtarget &Subtarget, in lowerShuffleWithUndefHalf() argument
17376 if (Subtarget.hasAVX2()) { in lowerShuffleWithUndefHalf()
17381 Subtarget.hasFastVariableCrossLaneShuffle())) in lowerShuffleWithUndefHalf()
17391 if (Subtarget.hasAVX512() && VT.is512BitVector()) in lowerShuffleWithUndefHalf()
17407 if (Subtarget.hasAVX2() && EltWidth == 64) in lowerShuffleWithUndefHalf()
17410 if (Subtarget.hasAVX512() && VT.is512BitVector()) in lowerShuffleWithUndefHalf()
17442 const X86Subtarget &Subtarget, SelectionDAG &DAG) { in lowerShuffleAsRepeatedMaskAndLanePermute() argument
17449 if (Subtarget.hasAVX2()) { in lowerShuffleAsRepeatedMaskAndLanePermute()
17611 if (Subtarget.hasAVX2() && VT.is256BitVector()) { in lowerShuffleAsRepeatedMaskAndLanePermute()
17617 if (Subtarget.hasBWI() && VT == MVT::v64i8) in lowerShuffleAsRepeatedMaskAndLanePermute()
17675 const X86Subtarget &Subtarget, in lowerShuffleWithSHUFPD() argument
17688 V1 = getZeroVector(VT, Subtarget, DAG, DL); in lowerShuffleWithSHUFPD()
17690 V2 = getZeroVector(VT, Subtarget, DAG, DL); in lowerShuffleWithSHUFPD()
17738 const X86Subtarget &Subtarget, in lowerV4F64Shuffle() argument
17745 Subtarget, DAG)) in lowerV4F64Shuffle()
17751 Mask, Subtarget, DAG)) in lowerV4F64Shuffle()
17768 if (Subtarget.hasAVX2()) in lowerV4F64Shuffle()
17775 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG)) in lowerV4F64Shuffle()
17780 Mask, DAG, Subtarget)) in lowerV4F64Shuffle()
17785 DAG, Subtarget); in lowerV4F64Shuffle()
17793 Zeroable, Subtarget, DAG)) in lowerV4F64Shuffle()
17798 Zeroable, Subtarget, DAG)) in lowerV4F64Shuffle()
17818 Subtarget, DAG); in lowerV4F64Shuffle()
17823 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG)) in lowerV4F64Shuffle()
17830 if (!(Subtarget.hasAVX2() && (V1IsInPlace || V2IsInPlace))) in lowerV4F64Shuffle()
17832 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG)) in lowerV4F64Shuffle()
17836 if (Subtarget.hasVLX()) in lowerV4F64Shuffle()
17838 DAG, Subtarget)) in lowerV4F64Shuffle()
17843 if (Subtarget.hasAVX2()) in lowerV4F64Shuffle()
17845 Subtarget, DAG); in lowerV4F64Shuffle()
17849 Subtarget, DAG); in lowerV4F64Shuffle()
17858 const X86Subtarget &Subtarget, in lowerV4I64Shuffle() argument
17863 assert(Subtarget.hasAVX2() && "We can only lower v4i64 with AVX2!"); in lowerV4I64Shuffle()
17866 Subtarget, DAG)) in lowerV4I64Shuffle()
17870 Zeroable, Subtarget, DAG)) in lowerV4I64Shuffle()
17875 Subtarget, DAG)) in lowerV4I64Shuffle()
17900 Zeroable, Subtarget, DAG)) in lowerV4I64Shuffle()
17904 if (Subtarget.hasVLX()) { in lowerV4I64Shuffle()
17906 Subtarget, DAG)) in lowerV4I64Shuffle()
17910 DAG, Subtarget)) in lowerV4I64Shuffle()
17916 Subtarget, DAG)) in lowerV4I64Shuffle()
17930 Subtarget, DAG); in lowerV4I64Shuffle()
17935 DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG)) in lowerV4I64Shuffle()
17949 DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG)) in lowerV4I64Shuffle()
17954 Subtarget, DAG); in lowerV4I64Shuffle()
17963 const X86Subtarget &Subtarget, in lowerV8F32Shuffle() argument
17970 Zeroable, Subtarget, DAG)) in lowerV8F32Shuffle()
17975 Subtarget, DAG)) in lowerV8F32Shuffle()
18007 DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG)) in lowerV8F32Shuffle()
18017 if (Subtarget.hasAVX2()) { in lowerV8F32Shuffle()
18023 DAG, Subtarget); in lowerV8F32Shuffle()
18029 DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG)) in lowerV8F32Shuffle()
18033 if (Subtarget.hasVLX()) in lowerV8F32Shuffle()
18035 DAG, Subtarget)) in lowerV8F32Shuffle()
18041 if (!Subtarget.hasAVX512() && isUnpackWdShuffleMask(Mask, MVT::v8f32, DAG)) in lowerV8F32Shuffle()
18042 return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, Subtarget, in lowerV8F32Shuffle()
18047 if (Subtarget.hasAVX2()) in lowerV8F32Shuffle()
18049 Subtarget, DAG); in lowerV8F32Shuffle()
18053 Subtarget, DAG); in lowerV8F32Shuffle()
18062 const X86Subtarget &Subtarget, in lowerV8I32Shuffle() argument
18067 assert(Subtarget.hasAVX2() && "We can only lower v8i32 with AVX2!"); in lowerV8I32Shuffle()
18073 Zeroable, Subtarget, DAG)) in lowerV8I32Shuffle()
18080 !Subtarget.hasAVX512()) in lowerV8I32Shuffle()
18081 return lowerShuffleAsSplitOrBlend(DL, MVT::v8i32, V1, V2, Mask, Subtarget, in lowerV8I32Shuffle()
18085 Zeroable, Subtarget, DAG)) in lowerV8I32Shuffle()
18090 Subtarget, DAG)) in lowerV8I32Shuffle()
18112 Zeroable, Subtarget, DAG)) in lowerV8I32Shuffle()
18116 if (Subtarget.hasVLX()) { in lowerV8I32Shuffle()
18118 Subtarget, DAG)) in lowerV8I32Shuffle()
18122 DAG, Subtarget)) in lowerV8I32Shuffle()
18128 Subtarget, DAG)) in lowerV8I32Shuffle()
18134 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG)) in lowerV8I32Shuffle()
18163 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG)) in lowerV8I32Shuffle()
18168 Subtarget, DAG); in lowerV8I32Shuffle()
18177 const X86Subtarget &Subtarget, in lowerV16I16Shuffle() argument
18182 assert(Subtarget.hasAVX2() && "We can only lower v16i16 with AVX2!"); in lowerV16I16Shuffle()
18188 DL, MVT::v16i16, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV16I16Shuffle()
18193 Subtarget, DAG)) in lowerV16I16Shuffle()
18197 Zeroable, Subtarget, DAG)) in lowerV16I16Shuffle()
18206 Subtarget)) in lowerV16I16Shuffle()
18211 Subtarget, DAG)) in lowerV16I16Shuffle()
18216 Zeroable, Subtarget, DAG)) in lowerV16I16Shuffle()
18221 Subtarget, DAG)) in lowerV16I16Shuffle()
18227 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG)) in lowerV16I16Shuffle()
18233 lowerShuffleAsBitRotate(DL, MVT::v16i16, V1, Mask, Subtarget, DAG)) in lowerV16I16Shuffle()
18245 DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget)) in lowerV16I16Shuffle()
18249 DAG, Subtarget); in lowerV16I16Shuffle()
18258 DL, MVT::v16i16, V1, RepeatedMask, Subtarget, DAG); in lowerV16I16Shuffle()
18263 Zeroable, Subtarget, DAG)) in lowerV16I16Shuffle()
18267 if (Subtarget.hasBWI()) in lowerV16I16Shuffle()
18268 return lowerShuffleWithPERMV(DL, MVT::v16i16, Mask, V1, V2, Subtarget, DAG); in lowerV16I16Shuffle()
18273 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG)) in lowerV16I16Shuffle()
18278 DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget)) in lowerV16I16Shuffle()
18283 Subtarget, DAG); in lowerV16I16Shuffle()
18292 const X86Subtarget &Subtarget, in lowerV32I8Shuffle() argument
18297 assert(Subtarget.hasAVX2() && "We can only lower v32i8 with AVX2!"); in lowerV32I8Shuffle()
18303 Zeroable, Subtarget, DAG)) in lowerV32I8Shuffle()
18308 Subtarget, DAG)) in lowerV32I8Shuffle()
18312 Zeroable, Subtarget, DAG)) in lowerV32I8Shuffle()
18321 Subtarget)) in lowerV32I8Shuffle()
18326 Subtarget, DAG)) in lowerV32I8Shuffle()
18331 Zeroable, Subtarget, DAG)) in lowerV32I8Shuffle()
18336 Subtarget, DAG)) in lowerV32I8Shuffle()
18342 lowerShuffleAsBitRotate(DL, MVT::v32i8, V1, Mask, Subtarget, DAG)) in lowerV32I8Shuffle()
18348 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG)) in lowerV32I8Shuffle()
18360 DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget)) in lowerV32I8Shuffle()
18364 DAG, Subtarget); in lowerV32I8Shuffle()
18368 Zeroable, Subtarget, DAG)) in lowerV32I8Shuffle()
18372 if (Subtarget.hasVBMI()) in lowerV32I8Shuffle()
18373 return lowerShuffleWithPERMV(DL, MVT::v32i8, Mask, V1, V2, Subtarget, DAG); in lowerV32I8Shuffle()
18378 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG)) in lowerV32I8Shuffle()
18383 DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget)) in lowerV32I8Shuffle()
18389 if (Subtarget.hasVLX()) in lowerV32I8Shuffle()
18396 Subtarget, DAG); in lowerV32I8Shuffle()
18406 const X86Subtarget &Subtarget, in lower256BitShuffle() argument
18415 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lower256BitShuffle()
18420 lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG)) in lower256BitShuffle()
18429 if (VT.isInteger() && !Subtarget.hasAVX2()) { in lower256BitShuffle()
18435 Subtarget, DAG)) in lower256BitShuffle()
18458 return lowerV4F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower256BitShuffle()
18460 return lowerV4I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower256BitShuffle()
18462 return lowerV8F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower256BitShuffle()
18464 return lowerV8I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower256BitShuffle()
18466 return lowerV16I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower256BitShuffle()
18468 return lowerV32I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower256BitShuffle()
18478 const X86Subtarget &Subtarget, in lowerV4X128Shuffle() argument
18501 getZeroVector(VT, Subtarget, DAG, DL), LoV, in lowerV4X128Shuffle()
18586 const X86Subtarget &Subtarget, in lowerV8F64Shuffle() argument
18615 V2, Subtarget, DAG)) in lowerV8F64Shuffle()
18623 Zeroable, Subtarget, DAG)) in lowerV8F64Shuffle()
18627 DAG, Subtarget)) in lowerV8F64Shuffle()
18631 Zeroable, Subtarget, DAG)) in lowerV8F64Shuffle()
18634 return lowerShuffleWithPERMV(DL, MVT::v8f64, Mask, V1, V2, Subtarget, DAG); in lowerV8F64Shuffle()
18640 const X86Subtarget &Subtarget, in lowerV16F32Shuffle() argument
18667 Zeroable, Subtarget, DAG)) in lowerV16F32Shuffle()
18677 DL, MVT::v16f32, V1, V2, Mask, Subtarget, DAG)) in lowerV16F32Shuffle()
18690 V1, V2, DAG, Subtarget)) in lowerV16F32Shuffle()
18693 return lowerShuffleWithPERMV(DL, MVT::v16f32, Mask, V1, V2, Subtarget, DAG); in lowerV16F32Shuffle()
18699 const X86Subtarget &Subtarget, in lowerV8I64Shuffle() argument
18727 V2, Subtarget, DAG)) in lowerV8I64Shuffle()
18732 Zeroable, Subtarget, DAG)) in lowerV8I64Shuffle()
18737 Subtarget, DAG)) in lowerV8I64Shuffle()
18741 if (Subtarget.hasBWI()) in lowerV8I64Shuffle()
18743 Subtarget, DAG)) in lowerV8I64Shuffle()
18751 DAG, Subtarget)) in lowerV8I64Shuffle()
18755 Zeroable, Subtarget, DAG)) in lowerV8I64Shuffle()
18758 return lowerShuffleWithPERMV(DL, MVT::v8i64, Mask, V1, V2, Subtarget, DAG); in lowerV8I64Shuffle()
18764 const X86Subtarget &Subtarget, in lowerV16I32Shuffle() argument
18774 DL, MVT::v16i32, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV16I32Shuffle()
18796 Zeroable, Subtarget, DAG)) in lowerV16I32Shuffle()
18801 Subtarget, DAG)) in lowerV16I32Shuffle()
18805 if (Subtarget.hasBWI()) in lowerV16I32Shuffle()
18807 Subtarget, DAG)) in lowerV16I32Shuffle()
18823 DL, MVT::v16i32, V1, V2, Mask, Subtarget, DAG)) in lowerV16I32Shuffle()
18828 DAG, Subtarget)) in lowerV16I32Shuffle()
18832 Zeroable, Subtarget, DAG)) in lowerV16I32Shuffle()
18835 return lowerShuffleWithPERMV(DL, MVT::v16i32, Mask, V1, V2, Subtarget, DAG); in lowerV16I32Shuffle()
18841 const X86Subtarget &Subtarget, in lowerV32I16Shuffle() argument
18846 assert(Subtarget.hasBWI() && "We can only lower v32i16 with AVX-512-BWI!"); in lowerV32I16Shuffle()
18852 DL, MVT::v32i16, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV32I16Shuffle()
18861 lowerShuffleWithPACK(DL, MVT::v32i16, Mask, V1, V2, DAG, Subtarget)) in lowerV32I16Shuffle()
18866 Zeroable, Subtarget, DAG)) in lowerV32I16Shuffle()
18871 Subtarget, DAG)) in lowerV32I16Shuffle()
18877 lowerShuffleAsBitRotate(DL, MVT::v32i16, V1, Mask, Subtarget, DAG)) in lowerV32I16Shuffle()
18886 RepeatedMask, Subtarget, DAG); in lowerV32I16Shuffle()
18891 Zeroable, Subtarget, DAG)) in lowerV32I16Shuffle()
18895 Zeroable, Subtarget, DAG)) in lowerV32I16Shuffle()
18898 return lowerShuffleWithPERMV(DL, MVT::v32i16, Mask, V1, V2, Subtarget, DAG); in lowerV32I16Shuffle()
18904 const X86Subtarget &Subtarget, in lowerV64I8Shuffle() argument
18909 assert(Subtarget.hasBWI() && "We can only lower v64i8 with AVX-512-BWI!"); in lowerV64I8Shuffle()
18915 DL, MVT::v64i8, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV64I8Shuffle()
18924 Subtarget)) in lowerV64I8Shuffle()
18929 Zeroable, Subtarget, DAG)) in lowerV64I8Shuffle()
18934 Subtarget, DAG)) in lowerV64I8Shuffle()
18940 lowerShuffleAsBitRotate(DL, MVT::v64i8, V1, Mask, Subtarget, DAG)) in lowerV64I8Shuffle()
18945 Zeroable, Subtarget, DAG)) in lowerV64I8Shuffle()
18949 Zeroable, Subtarget, DAG)) in lowerV64I8Shuffle()
18955 DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG)) in lowerV64I8Shuffle()
18959 DL, MVT::v64i8, V1, V2, Mask, DAG, Subtarget)) in lowerV64I8Shuffle()
18963 Zeroable, Subtarget, DAG)) in lowerV64I8Shuffle()
18970 Mask, Subtarget, DAG)) in lowerV64I8Shuffle()
18984 DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG)) in lowerV64I8Shuffle()
18988 if (Subtarget.hasVBMI()) in lowerV64I8Shuffle()
18989 return lowerShuffleWithPERMV(DL, MVT::v64i8, Mask, V1, V2, Subtarget, DAG); in lowerV64I8Shuffle()
19002 const X86Subtarget &Subtarget, in lower512BitShuffle() argument
19004 assert(Subtarget.hasAVX512() && in lower512BitShuffle()
19014 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lower512BitShuffle()
19019 lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG)) in lower512BitShuffle()
19024 Subtarget, DAG)) in lower512BitShuffle()
19027 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) { in lower512BitShuffle()
19031 Subtarget, DAG)) in lower512BitShuffle()
19052 return lowerV8F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower512BitShuffle()
19054 return lowerV16F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower512BitShuffle()
19056 return lowerV8I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower512BitShuffle()
19058 return lowerV16I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower512BitShuffle()
19060 return lowerV32I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower512BitShuffle()
19062 return lowerV64I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower512BitShuffle()
19071 const X86Subtarget &Subtarget, in lower1BitShuffleAsKSHIFTR() argument
19101 if ((!Subtarget.hasDQI() && NumElts == 8) || NumElts < 8) in lower1BitShuffleAsKSHIFTR()
19102 WideVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1; in lower1BitShuffleAsKSHIFTR()
19152 const X86Subtarget &Subtarget, in lower1BitShuffle() argument
19154 assert(Subtarget.hasAVX512() && in lower1BitShuffle()
19193 if (SDValue Shift = lower1BitShuffleAsKSHIFTR(DL, Mask, VT, V1, V2, Subtarget, in lower1BitShuffle()
19204 if ((!Subtarget.hasDQI() && NumElts == 8) || NumElts < 8) in lower1BitShuffle()
19205 WideVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1; in lower1BitShuffle()
19253 ExtVT = Subtarget.hasVLX() ? MVT::v8i32 : MVT::v8i64; in lower1BitShuffle()
19258 ExtVT = Subtarget.canExtendTo512DQ() ? MVT::v16i32 : MVT::v16i16; in lower1BitShuffle()
19263 assert(Subtarget.hasBWI() && "Expected AVX512BW support"); in lower1BitShuffle()
19264 ExtVT = Subtarget.canExtendTo512BW() ? MVT::v32i16 : MVT::v32i8; in lower1BitShuffle()
19269 if (!Subtarget.useBWIRegs()) in lower1BitShuffle()
19281 if ((Subtarget.hasBWI() && (NumElems >= 32)) || in lower1BitShuffle()
19282 (Subtarget.hasDQI() && (NumElems < 32))) in lower1BitShuffle()
19357 const X86Subtarget &Subtarget);
19366 static SDValue lowerVECTOR_SHUFFLE(SDValue Op, const X86Subtarget &Subtarget, in lowerVECTOR_SHUFFLE() argument
19418 return getZeroVector(VT, Subtarget, DAG, DL); in lowerVECTOR_SHUFFLE()
19434 Subtarget, DAG)) in lowerVECTOR_SHUFFLE()
19459 V2 = getZeroVector(NewVT, Subtarget, DAG, DL); in lowerVECTOR_SHUFFLE()
19474 Ops, Mask, VT.getSizeInBits(), DL, DAG, Subtarget)) in lowerVECTOR_SHUFFLE()
19491 return lower128BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG); in lowerVECTOR_SHUFFLE()
19494 return lower256BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG); in lowerVECTOR_SHUFFLE()
19497 return lower512BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG); in lowerVECTOR_SHUFFLE()
19500 return lower1BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG); in lowerVECTOR_SHUFFLE()
19507 const X86Subtarget &Subtarget, in lowerVSELECTtoVectorShuffle() argument
19548 if (SDValue BlendOp = lowerVSELECTtoVectorShuffle(Op, Subtarget, DAG)) in LowerVSELECT()
19559 if (!Subtarget.hasSSE41()) in LowerVSELECT()
19566 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) in LowerVSELECT()
19604 if (Subtarget.hasAVX2()) in LowerVSELECT()
19674 const X86Subtarget &Subtarget) { in ExtractBitFromMaskVector() argument
19682 assert((VecVT.getVectorNumElements() <= 16 || Subtarget.hasBWI()) && in ExtractBitFromMaskVector()
19705 if ((!Subtarget.hasDQI() && NumElems == 8) || NumElems < 8) { in ExtractBitFromMaskVector()
19706 WideVecVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1; in ExtractBitFromMaskVector()
19730 return ExtractBitFromMaskVector(Op, DAG, Subtarget); in LowerEXTRACT_VECTOR_ELT()
19793 !(Subtarget.hasSSE41() && X86::mayFoldIntoStore(Op))) { in LowerEXTRACT_VECTOR_ELT()
19794 if (Subtarget.hasFP16()) in LowerEXTRACT_VECTOR_ELT()
19807 if (Subtarget.hasSSE41()) in LowerEXTRACT_VECTOR_ELT()
19873 const X86Subtarget &Subtarget) { in InsertBitToMaskVector() argument
19905 return InsertBitToMaskVector(Op, DAG, Subtarget); in LowerINSERT_VECTOR_ELT()
19917 if (!(Subtarget.hasBWI() || in LowerINSERT_VECTOR_ELT()
19918 (Subtarget.hasAVX512() && EltSizeInBits >= 32) || in LowerINSERT_VECTOR_ELT()
19919 (Subtarget.hasSSE41() && VT.isFloatingPoint()))) in LowerINSERT_VECTOR_ELT()
19952 ((VT == MVT::v16i8 && !Subtarget.hasSSE41()) || in LowerINSERT_VECTOR_ELT()
19953 ((VT == MVT::v32i8 || VT == MVT::v16i16) && !Subtarget.hasInt256()))) { in LowerINSERT_VECTOR_ELT()
19963 if (Subtarget.hasSSE41() && in LowerINSERT_VECTOR_ELT()
19968 SDValue CstVector = IsZeroElt ? getZeroVector(VT, Subtarget, DAG, dl) in LowerINSERT_VECTOR_ELT()
19983 if ((Subtarget.hasAVX() && (EltVT == MVT::f64 || EltVT == MVT::f32)) || in LowerINSERT_VECTOR_ELT()
19984 (Subtarget.hasAVX2() && (EltVT == MVT::i32 || EltVT == MVT::i64))) { in LowerINSERT_VECTOR_ELT()
19999 ((Subtarget.hasAVX2() && EltSizeInBits != 8) || in LowerINSERT_VECTOR_ELT()
20000 (Subtarget.hasAVX() && (EltSizeInBits >= 32) && in LowerINSERT_VECTOR_ELT()
20001 X86::mayFoldLoad(N1, Subtarget)))) { in LowerINSERT_VECTOR_ELT()
20029 return getShuffleVectorZeroOrUndef(N1, 0, true, Subtarget, DAG); in LowerINSERT_VECTOR_ELT()
20038 N1 = getShuffleVectorZeroOrUndef(N1, 0, true, Subtarget, DAG); in LowerINSERT_VECTOR_ELT()
20045 if (VT == MVT::v8i16 || (VT == MVT::v16i8 && Subtarget.hasSSE41())) { in LowerINSERT_VECTOR_ELT()
20048 assert(Subtarget.hasSSE2() && "SSE2 required for PINSRW"); in LowerINSERT_VECTOR_ELT()
20052 assert(Subtarget.hasSSE41() && "SSE41 required for PINSRB"); in LowerINSERT_VECTOR_ELT()
20062 if (Subtarget.hasSSE41()) { in LowerINSERT_VECTOR_ELT()
20074 if (IdxVal == 0 && (!MinSize || !X86::mayFoldLoad(N1, Subtarget))) { in LowerINSERT_VECTOR_ELT()
20100 static SDValue LowerSCALAR_TO_VECTOR(SDValue Op, const X86Subtarget &Subtarget, in LowerSCALAR_TO_VECTOR() argument
20108 return getZeroVector(OpVT, Subtarget, DAG, dl); in LowerSCALAR_TO_VECTOR()
20128 if (OpVT == MVT::v4i32 || (OpVT == MVT::v8i16 && Subtarget.hasFP16())) in LowerSCALAR_TO_VECTOR()
20139 static SDValue LowerINSERT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget, in LowerINSERT_SUBVECTOR() argument
20143 return insert1BitVector(Op, DAG, Subtarget); in LowerINSERT_SUBVECTOR()
20146 static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget, in LowerEXTRACT_SUBVECTOR() argument
20163 if ((!Subtarget.hasDQI() && NumElems == 8) || NumElems < 8) { in LowerEXTRACT_SUBVECTOR()
20164 WideVecVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1; in LowerEXTRACT_SUBVECTOR()
20186 if (Subtarget.isPICStyleRIPRel() && in getGlobalWrapperKind()
20209 unsigned char OpFlag = Subtarget.classifyLocalReference(nullptr); in LowerConstantPool()
20231 unsigned char OpFlag = Subtarget.classifyLocalReference(nullptr); in LowerJumpTable()
20256 Subtarget.classifyBlockAddressReference(); in LowerBlockAddress()
20294 OpFlags = Subtarget.classifyGlobalFunctionReference(GV, Mod); in LowerGlobalOrExternal()
20296 OpFlags = Subtarget.classifyGlobalReference(GV, Mod); in LowerGlobalOrExternal()
20521 if (Subtarget.isTargetELF()) { in LowerGlobalTLSAddress()
20525 if (Subtarget.is64Bit()) { in LowerGlobalTLSAddress()
20526 if (Subtarget.isTarget64BitLP64()) in LowerGlobalTLSAddress()
20532 return LowerToTLSLocalDynamicModel(GA, DAG, PtrVT, Subtarget.is64Bit(), in LowerGlobalTLSAddress()
20533 Subtarget.isTarget64BitLP64()); in LowerGlobalTLSAddress()
20536 return LowerToTLSExecModel(GA, DAG, PtrVT, model, Subtarget.is64Bit(), in LowerGlobalTLSAddress()
20542 if (Subtarget.isTargetDarwin()) { in LowerGlobalTLSAddress()
20545 unsigned WrapperKind = Subtarget.isPICStyleRIPRel() ? in LowerGlobalTLSAddress()
20550 bool PIC32 = PositionIndependent && !Subtarget.is64Bit(); in LowerGlobalTLSAddress()
20584 unsigned Reg = Subtarget.is64Bit() ? X86::RAX : X86::EAX; in LowerGlobalTLSAddress()
20588 if (Subtarget.isOSWindows()) { in LowerGlobalTLSAddress()
20606 Value *Ptr = Constant::getNullValue(Subtarget.is64Bit() in LowerGlobalTLSAddress()
20612 SDValue TlsArray = Subtarget.is64Bit() in LowerGlobalTLSAddress()
20614 : (Subtarget.isTargetWindowsGNU() in LowerGlobalTLSAddress()
20627 if (Subtarget.is64Bit()) in LowerGlobalTLSAddress()
20669 const X86Subtarget &Subtarget) { in LowerI64IntToFP_AVX512DQ() argument
20681 if (!Subtarget.hasDQI() || SrcVT != MVT::i64 || Subtarget.is64Bit() || in LowerI64IntToFP_AVX512DQ()
20688 unsigned NumElts = Subtarget.hasVLX() ? 4 : 8; in LowerI64IntToFP_AVX512DQ()
20711 const X86Subtarget &Subtarget) { in LowerI64IntToFP16() argument
20722 if (SrcVT != MVT::i64 || Subtarget.is64Bit() || VT != MVT::f16) in LowerI64IntToFP16()
20727 assert(Subtarget.hasFP16() && "Expected FP16"); in LowerI64IntToFP16()
20747 const X86Subtarget &Subtarget) { in useVectorCast() argument
20751 if (!Subtarget.hasSSE2() || FromVT != MVT::v4i32) in useVectorCast()
20754 return ToVT == MVT::v4f32 || (Subtarget.hasAVX() && ToVT == MVT::v4f64); in useVectorCast()
20758 if (!Subtarget.hasAVX512() || FromVT != MVT::v4i32) in useVectorCast()
20772 const X86Subtarget &Subtarget) { in vectorizeExtractedCast() argument
20787 if (!useVectorCast(Cast.getOpcode(), Vec128VT, ToVT, Subtarget)) in vectorizeExtractedCast()
20814 const X86Subtarget &Subtarget) { in lowerFPToIntToFP() argument
20829 if (!Subtarget.hasSSE2() || (VT != MVT::f32 && VT != MVT::f64) || in lowerFPToIntToFP()
20861 const X86Subtarget &Subtarget) { in lowerINT_TO_FP_vXi64() argument
20867 if (Subtarget.hasDQI()) { in lowerINT_TO_FP_vXi64()
20868 assert(!Subtarget.hasVLX() && "Unexpected features"); in lowerINT_TO_FP_vXi64()
20969 const X86Subtarget &Subtarget) { in isLegalConversion() argument
20970 if (VT == MVT::v4i32 && Subtarget.hasSSE2() && IsSigned) in isLegalConversion()
20972 if (VT == MVT::v8i32 && Subtarget.hasAVX() && IsSigned) in isLegalConversion()
20974 if (Subtarget.hasVLX() && (VT == MVT::v4i32 || VT == MVT::v8i32)) in isLegalConversion()
20976 if (Subtarget.useAVX512Regs()) { in isLegalConversion()
20979 if (VT == MVT::v8i64 && Subtarget.hasDQI()) in isLegalConversion()
20982 if (Subtarget.hasDQI() && Subtarget.hasVLX() && in isLegalConversion()
21000 else if (isLegalConversion(SrcVT, true, Subtarget)) in LowerSINT_TO_FP()
21003 if (Subtarget.isTargetWin64() && SrcVT == MVT::i128) in LowerSINT_TO_FP()
21006 if (SDValue Extract = vectorizeExtractedCast(Op, DAG, Subtarget)) in LowerSINT_TO_FP()
21009 if (SDValue R = lowerFPToIntToFP(Op, DAG, Subtarget)) in LowerSINT_TO_FP()
21026 return lowerINT_TO_FP_vXi64(Op, DAG, Subtarget); in LowerSINT_TO_FP()
21040 if (SrcVT == MVT::i64 && UseSSEReg && Subtarget.is64Bit()) in LowerSINT_TO_FP()
21043 if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, DAG, Subtarget)) in LowerSINT_TO_FP()
21045 if (SDValue V = LowerI64IntToFP16(Op, DAG, Subtarget)) in LowerSINT_TO_FP()
21062 if (SrcVT == MVT::i64 && Subtarget.hasSSE2() && !Subtarget.is64Bit()) in LowerSINT_TO_FP()
21131 const X86Subtarget &Subtarget) { in shouldUseHorizontalOp() argument
21133 bool HasFastHOps = Subtarget.hasFastHorizontalOps(); in shouldUseHorizontalOp()
21139 const X86Subtarget &Subtarget) { in LowerUINT_TO_FP_i64() argument
21193 if (Subtarget.hasSSE3() && in LowerUINT_TO_FP_i64()
21194 shouldUseHorizontalOp(true, DAG, Subtarget)) { in LowerUINT_TO_FP_i64()
21207 const X86Subtarget &Subtarget) { in LowerUINT_TO_FP_i32() argument
21219 Load = getShuffleVectorZeroOrUndef(Load, 0, true, Subtarget, DAG); in LowerUINT_TO_FP_i32()
21257 const X86Subtarget &Subtarget, in lowerUINT_TO_FP_v2i32() argument
21267 if (Subtarget.hasAVX512()) { in lowerUINT_TO_FP_v2i32()
21268 if (!Subtarget.hasVLX()) { in lowerUINT_TO_FP_v2i32()
21310 const X86Subtarget &Subtarget) { in lowerUINT_TO_FP_vXi32() argument
21318 if (Subtarget.hasAVX512()) { in lowerUINT_TO_FP_vXi32()
21320 assert(!Subtarget.hasVLX() && "Unexpected features"); in lowerUINT_TO_FP_vXi32()
21354 if (Subtarget.hasAVX() && VecIntVT == MVT::v4i32 && in lowerUINT_TO_FP_vXi32()
21415 if (Subtarget.hasSSE41()) { in lowerUINT_TO_FP_vXi32()
21468 const X86Subtarget &Subtarget) { in lowerUINT_TO_FP_vec() argument
21478 return lowerUINT_TO_FP_v2i32(Op, DAG, Subtarget, dl); in lowerUINT_TO_FP_vec()
21481 return lowerUINT_TO_FP_vXi32(Op, DAG, Subtarget); in lowerUINT_TO_FP_vec()
21484 return lowerINT_TO_FP_vXi64(Op, DAG, Subtarget); in lowerUINT_TO_FP_vec()
21505 else if (isLegalConversion(SrcVT, false, Subtarget)) in LowerUINT_TO_FP()
21509 return lowerUINT_TO_FP_vec(Op, DAG, Subtarget); in LowerUINT_TO_FP()
21511 if (Subtarget.isTargetWin64() && SrcVT == MVT::i128) in LowerUINT_TO_FP()
21514 if (SDValue Extract = vectorizeExtractedCast(Op, DAG, Subtarget)) in LowerUINT_TO_FP()
21517 if (Subtarget.hasAVX512() && isScalarFPTypeInSSEReg(DstVT) && in LowerUINT_TO_FP()
21518 (SrcVT == MVT::i32 || (SrcVT == MVT::i64 && Subtarget.is64Bit()))) { in LowerUINT_TO_FP()
21525 if (SrcVT == MVT::i32 && Subtarget.is64Bit()) { in LowerUINT_TO_FP()
21533 if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, DAG, Subtarget)) in LowerUINT_TO_FP()
21535 if (SDValue V = LowerI64IntToFP16(Op, DAG, Subtarget)) in LowerUINT_TO_FP()
21540 if (SrcVT == MVT::i64 && DstVT == MVT::f64 && Subtarget.hasSSE2() && in LowerUINT_TO_FP()
21542 return LowerUINT_TO_FP_i64(Op, DAG, Subtarget); in LowerUINT_TO_FP()
21545 if (SrcVT == MVT::i32 && Subtarget.hasSSE2() && DstVT != MVT::f80 && in LowerUINT_TO_FP()
21547 return LowerUINT_TO_FP_i32(Op, DAG, Subtarget); in LowerUINT_TO_FP()
21548 if (Subtarget.is64Bit() && SrcVT == MVT::i64 && in LowerUINT_TO_FP()
21574 if (isScalarFPTypeInSSEReg(Op.getValueType()) && !Subtarget.is64Bit()) { in LowerUINT_TO_FP()
21793 const X86Subtarget &Subtarget) { in LowerAVXExtend() argument
21816 if (VT == MVT::v32i16 && !Subtarget.hasBWI()) { in LowerAVXExtend()
21821 if (Subtarget.hasInt256()) in LowerAVXExtend()
21869 const X86Subtarget &Subtarget, in LowerZERO_EXTEND_Mask() argument
21888 if (!Subtarget.hasBWI()) { in LowerZERO_EXTEND_Mask()
21890 if (NumElts == 16 && !Subtarget.canExtendTo512DQ()) in LowerZERO_EXTEND_Mask()
21898 if (!ExtVT.is512BitVector() && !Subtarget.hasVLX()) { in LowerZERO_EXTEND_Mask()
21926 static SDValue LowerZERO_EXTEND(SDValue Op, const X86Subtarget &Subtarget, in LowerZERO_EXTEND() argument
21932 return LowerZERO_EXTEND_Mask(Op, Subtarget, DAG); in LowerZERO_EXTEND()
21934 assert(Subtarget.hasAVX() && "Expected AVX support"); in LowerZERO_EXTEND()
21935 return LowerAVXExtend(Op, DAG, Subtarget); in LowerZERO_EXTEND()
21945 const X86Subtarget &Subtarget) { in truncateVectorWithPACK() argument
21951 if (!Subtarget.hasSSE2()) in truncateVectorWithPACK()
21981 (Opcode == X86ISD::PACKSS || Subtarget.hasSSE41())) { in truncateVectorWithPACK()
22014 if (SrcVT.is512BitVector() && Subtarget.hasInt256()) { in truncateVectorWithPACK()
22033 return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget); in truncateVectorWithPACK()
22039 Lo = truncateVectorWithPACK(Opcode, PackedVT, Lo, DL, DAG, Subtarget); in truncateVectorWithPACK()
22040 Hi = truncateVectorWithPACK(Opcode, PackedVT, Hi, DL, DAG, Subtarget); in truncateVectorWithPACK()
22044 return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget); in truncateVectorWithPACK()
22048 const X86Subtarget &Subtarget) { in LowerTruncateVecI1() argument
22060 if (Subtarget.hasBWI()) { in LowerTruncateVecI1()
22088 if (NumElts == 16 && !Subtarget.canExtendTo512DQ()) { in LowerTruncateVecI1()
22110 MVT EltVT = Subtarget.hasVLX() ? MVT::i32 : MVT::getIntegerVT(512/NumElts); in LowerTruncateVecI1()
22123 if (Subtarget.hasDQI()) in LowerTruncateVecI1()
22143 assert((InVT == MVT::v16i64 || Subtarget.hasVLX()) && in LowerTRUNCATE()
22164 return LowerTruncateVecI1(Op, DAG, Subtarget); in LowerTRUNCATE()
22167 if (Subtarget.hasAVX512()) { in LowerTRUNCATE()
22168 if (InVT == MVT::v32i16 && !Subtarget.hasBWI()) { in LowerTRUNCATE()
22177 if (InVT != MVT::v16i16 || Subtarget.hasBWI() || in LowerTRUNCATE()
22178 Subtarget.canExtendTo512DQ()) in LowerTRUNCATE()
22183 unsigned NumPackedZeroBits = Subtarget.hasSSE41() ? NumPackedSignBits : 8; in LowerTRUNCATE()
22191 truncateVectorWithPACK(X86ISD::PACKUS, VT, In, DL, DAG, Subtarget)) in LowerTRUNCATE()
22198 truncateVectorWithPACK(X86ISD::PACKSS, VT, In, DL, DAG, Subtarget)) in LowerTRUNCATE()
22206 if (Subtarget.hasInt256()) { in LowerTRUNCATE()
22225 if (Subtarget.hasInt256()) { in LowerTRUNCATE()
22283 const X86Subtarget &Subtarget) { in expandFP_TO_UINT_SSE() argument
22305 if (VT == MVT::v8i32 && !Subtarget.hasAVX2()) { in expandFP_TO_UINT_SSE()
22336 } else if (isTypeLegal(SrcVT) && isLegalConversion(VT, IsSigned, Subtarget)) { in LowerFP_TO_INT()
22350 if (!IsSigned && !Subtarget.hasVLX()) { in LowerFP_TO_INT()
22351 assert(Subtarget.useAVX512Regs() && "Unexpected features!"); in LowerFP_TO_INT()
22379 if (Subtarget.hasFP16() && SrcVT.getVectorElementType() == MVT::f16) { in LowerFP_TO_INT()
22443 assert(Subtarget.useAVX512Regs() && "Requires avx512f"); in LowerFP_TO_INT()
22450 Subtarget.useAVX512Regs()) { in LowerFP_TO_INT()
22452 assert(!Subtarget.hasVLX() && "Unexpected features!"); in LowerFP_TO_INT()
22482 Subtarget.useAVX512Regs() && Subtarget.hasDQI()) { in LowerFP_TO_INT()
22483 assert(!Subtarget.hasVLX() && "Unexpected features!"); in LowerFP_TO_INT()
22510 if (!Subtarget.hasVLX()) { in LowerFP_TO_INT()
22527 assert(Subtarget.hasDQI() && Subtarget.hasVLX() && "Requires AVX512DQVL"); in LowerFP_TO_INT()
22545 return expandFP_TO_UINT_SSE(VT, Src, dl, DAG, Subtarget); in LowerFP_TO_INT()
22557 if (Subtarget.hasAVX512()) in LowerFP_TO_INT()
22562 if (!IsStrict && ((VT == MVT::i32 && !Subtarget.is64Bit()) || in LowerFP_TO_INT()
22563 (VT == MVT::i64 && Subtarget.is64Bit()))) { in LowerFP_TO_INT()
22602 if (Subtarget.is64Bit()) { in LowerFP_TO_INT()
22618 if (!Subtarget.hasSSE3()) in LowerFP_TO_INT()
22769 if (SatWidth == 32 && !IsSigned && Subtarget.is64Bit()) { in LowerFP_TO_INT_SAT()
22894 if (Subtarget.hasFP16()) in LowerFP_EXTEND()
22908 if (!Subtarget.hasF16C()) in LowerFP_EXTEND()
22913 getZeroVector(MVT::v8i16, Subtarget, DAG, DL), In, in LowerFP_EXTEND()
22935 assert(Subtarget.hasF16C() && "Unexpected features!"); in LowerFP_EXTEND()
22971 if (VT.getScalarType() == MVT::f16 && !Subtarget.hasFP16()) { in LowerFP_ROUND()
22972 if (!Subtarget.hasF16C() || SVT.getScalarType() != MVT::f32) in LowerFP_ROUND()
23083 const X86Subtarget &Subtarget) { in lowerAddSubToHorizontalOp() argument
23092 if (IsFP && !Subtarget.hasSSE3()) in lowerAddSubToHorizontalOp()
23094 if (!IsFP && !Subtarget.hasSSSE3()) in lowerAddSubToHorizontalOp()
23103 !shouldUseHorizontalOp(true, DAG, Subtarget)) in lowerAddSubToHorizontalOp()
23157 return lowerAddSubToHorizontalOp(Op, DAG, Subtarget); in lowerFaddFsub()
23457 const X86Subtarget &Subtarget, in LowerVectorAllZero() argument
23492 unsigned TestSize = Subtarget.hasAVX() ? 256 : 128; in LowerVectorAllZero()
23499 bool UsePTEST = Subtarget.hasSSE41(); in LowerVectorAllZero()
23513 getZeroVector(MVT::v16i8, Subtarget, DAG, DL)); in LowerVectorAllZero()
23523 const X86Subtarget &Subtarget, in MatchVectorAllZeroTest() argument
23527 if (!Subtarget.hasSSE2() || !Op->hasOneUse()) in MatchVectorAllZeroTest()
23572 if (SDValue V = LowerVectorAllZero(DL, VecIns.back(), CC, Mask, Subtarget, in MatchVectorAllZeroTest()
23585 LowerVectorAllZero(DL, Match, CC, Mask, Subtarget, DAG, CCode)) { in MatchVectorAllZeroTest()
23630 SelectionDAG &DAG, const X86Subtarget &Subtarget) { in EmitTest() argument
23739 const X86Subtarget &Subtarget) { in EmitCmp() argument
23741 return EmitTest(Op0, X86CC, dl, DAG, Subtarget); in EmitCmp()
23750 if (CmpVT == MVT::i16 && !Subtarget.isAtom() && in EmitCmp()
23825 return Subtarget.hasFastVectorFSQRT(); in isFsqrtCheap()
23826 return Subtarget.hasFastScalarFSQRT(); in isFsqrtCheap()
23847 if ((VT == MVT::f32 && Subtarget.hasSSE1()) || in getSqrtEstimate()
23848 (VT == MVT::v4f32 && Subtarget.hasSSE1() && Reciprocal) || in getSqrtEstimate()
23849 (VT == MVT::v4f32 && Subtarget.hasSSE2() && !Reciprocal) || in getSqrtEstimate()
23850 (VT == MVT::v8f32 && Subtarget.hasAVX()) || in getSqrtEstimate()
23851 (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) { in getSqrtEstimate()
23865 Subtarget.hasFP16()) { in getSqrtEstimate()
23898 if ((VT == MVT::f32 && Subtarget.hasSSE1()) || in getRecipEstimate()
23899 (VT == MVT::v4f32 && Subtarget.hasSSE1()) || in getRecipEstimate()
23900 (VT == MVT::v8f32 && Subtarget.hasAVX()) || in getRecipEstimate()
23901 (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) { in getRecipEstimate()
23917 Subtarget.hasFP16()) { in getRecipEstimate()
23957 if (!Subtarget.canUseCMOV()) in BuildSDIVPow2()
23964 !(Subtarget.is64Bit() && VT == MVT::i64)) in BuildSDIVPow2()
24220 const X86Subtarget &Subtarget, in LowerVSETCCWithSUBUS() argument
24222 if (!Subtarget.hasSSE2()) in LowerVSETCCWithSUBUS()
24238 if (Subtarget.hasAVX()) in LowerVSETCCWithSUBUS()
24271 static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget, in LowerVSETCC() argument
24286 if (isSoftFP16(EltVT, Subtarget)) in LowerVSETCC()
24297 if (Subtarget.hasAVX512() && VT.getVectorElementType() == MVT::i1 && in LowerVSETCC()
24298 (!IsStrict || Subtarget.hasVLX() || in LowerVSETCC()
24316 if (!Subtarget.hasAVX()) { in LowerVSETCC()
24430 assert((Subtarget.hasAVX512() || (VT == VTOp0)) && in LowerVSETCC()
24437 assert((VTOp0.getScalarSizeInBits() >= 32 || Subtarget.hasBWI()) && in LowerVSETCC()
24443 if (VT.is128BitVector() && Subtarget.hasXOP()) { in LowerVSETCC()
24504 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerVSETCC()
24579 LowerVSETCCWithSUBUS(Op0, Op1, VT, Cond, dl, Subtarget, DAG)) in LowerVSETCC()
24598 if (Opc == X86ISD::PCMPGT && !Subtarget.hasSSE42()) { in LowerVSETCC()
24599 assert(Subtarget.hasSSE2() && "Don't know how to lower!"); in LowerVSETCC()
24659 if (Opc == X86ISD::PCMPEQ && !Subtarget.hasSSE41()) { in LowerVSETCC()
24662 assert(Subtarget.hasSSE2() && !FlipSigns && "Don't know how to lower!"); in LowerVSETCC()
24705 const X86Subtarget &Subtarget, in EmitAVX512Test() argument
24717 if (!(Subtarget.hasAVX512() && VT == MVT::v16i1) && in EmitAVX512Test()
24718 !(Subtarget.hasDQI() && VT == MVT::v8i1) && in EmitAVX512Test()
24719 !(Subtarget.hasBWI() && (VT == MVT::v32i1 || VT == MVT::v64i1))) in EmitAVX512Test()
24733 if (Subtarget.hasDQI() && (VT == MVT::v8i1 || VT == MVT::v16i1)) in EmitAVX512Test()
24735 if (Subtarget.hasBWI() && (VT == MVT::v32i1 || VT == MVT::v64i1)) in EmitAVX512Test()
24781 MatchVectorAllZeroTest(Op0, CC, dl, Subtarget, DAG, X86CC)) in emitFlagsForSetcc()
24785 if (SDValue Test = EmitAVX512Test(Op0, Op1, CC, dl, DAG, Subtarget, X86CC)) in emitFlagsForSetcc()
24828 SDValue EFLAGS = EmitCmp(Op0, Op1, CondCode, dl, DAG, Subtarget); in emitFlagsForSetcc()
24839 if (VT.isVector()) return LowerVSETCC(Op, Subtarget, DAG); in LowerSETCC()
25060 if (Subtarget.hasAVX512()) { in LowerSELECT()
25068 if (SSECC < 8 || Subtarget.hasAVX()) { in LowerSELECT()
25084 if (Subtarget.hasAVX() && !isNullFPConstant(Op1) && in LowerSELECT()
25108 if (isScalarFPTypeInSSEReg(VT) && Subtarget.hasAVX512()) { in LowerSELECT()
25149 if (Subtarget.canUseCMOV() && (VT == MVT::i32 || VT == MVT::i64) && in LowerSELECT()
25177 } else if (!Subtarget.canUseCMOV() && CondCode == X86::COND_E && in LowerSELECT()
25248 !isScalarFPTypeInSSEReg(VT) && Subtarget.canUseCMOV()) // FPStack? in LowerSELECT()
25286 Cond = EmitTest(Cond, X86::COND_NE, DL, DAG, Subtarget); in LowerSELECT()
25329 if ((Op.getValueType() == MVT::i8 && Subtarget.canUseCMOV()) || in LowerSELECT()
25330 (Op.getValueType() == MVT::i16 && !X86::mayFoldLoad(Op1, Subtarget) && in LowerSELECT()
25331 !X86::mayFoldLoad(Op2, Subtarget))) { in LowerSELECT()
25346 const X86Subtarget &Subtarget, in LowerSIGN_EXTEND_Mask() argument
25359 if (!Subtarget.hasBWI() && VTElt.getSizeInBits() <= 16) { in LowerSIGN_EXTEND_Mask()
25361 if (NumElts == 16 && !Subtarget.canExtendTo512DQ()) in LowerSIGN_EXTEND_Mask()
25369 if (!ExtVT.is512BitVector() && !Subtarget.hasVLX()) { in LowerSIGN_EXTEND_Mask()
25379 if ((Subtarget.hasDQI() && WideEltVT.getSizeInBits() >= 32) || in LowerSIGN_EXTEND_Mask()
25380 (Subtarget.hasBWI() && WideEltVT.getSizeInBits() <= 16)) { in LowerSIGN_EXTEND_Mask()
25402 static SDValue LowerANY_EXTEND(SDValue Op, const X86Subtarget &Subtarget, in LowerANY_EXTEND() argument
25408 return LowerSIGN_EXTEND_Mask(Op, Subtarget, DAG); in LowerANY_EXTEND()
25410 assert(Subtarget.hasAVX() && "Expected AVX support"); in LowerANY_EXTEND()
25411 return LowerAVXExtend(Op, DAG, Subtarget); in LowerANY_EXTEND()
25419 const X86Subtarget &Subtarget, in LowerEXTEND_VECTOR_INREG() argument
25433 if (!(VT.is128BitVector() && Subtarget.hasSSE2()) && in LowerEXTEND_VECTOR_INREG()
25434 !(VT.is256BitVector() && Subtarget.hasAVX()) && in LowerEXTEND_VECTOR_INREG()
25435 !(VT.is512BitVector() && Subtarget.hasAVX512())) in LowerEXTEND_VECTOR_INREG()
25455 if (Subtarget.hasInt256()) { in LowerEXTEND_VECTOR_INREG()
25470 if (Subtarget.hasAVX()) { in LowerEXTEND_VECTOR_INREG()
25530 static SDValue LowerSIGN_EXTEND(SDValue Op, const X86Subtarget &Subtarget, in LowerSIGN_EXTEND() argument
25538 return LowerSIGN_EXTEND_Mask(Op, Subtarget, DAG); in LowerSIGN_EXTEND()
25552 if (VT == MVT::v32i16 && !Subtarget.hasBWI()) { in LowerSIGN_EXTEND()
25557 if (Subtarget.hasInt256()) in LowerSIGN_EXTEND()
25651 static SDValue LowerStore(SDValue Op, const X86Subtarget &Subtarget, in LowerStore() argument
25663 assert(Subtarget.hasAVX512() && !Subtarget.hasDQI() && in LowerStore()
25692 !Subtarget.hasBWI())) { in LowerStore()
25713 if (Subtarget.hasSSE2()) { in LowerStore()
25716 MVT StVT = Subtarget.is64Bit() && StoreVT.isInteger() ? MVT::i64 : MVT::f64; in LowerStore()
25726 assert(Subtarget.hasSSE1() && "Expected SSE"); in LowerStore()
25740 static SDValue LowerLoad(SDValue Op, const X86Subtarget &Subtarget, in LowerLoad() argument
25754 assert(Subtarget.hasAVX512() && !Subtarget.hasDQI() && in LowerLoad()
25916 bool Lower = (Subtarget.isOSWindows() && !Subtarget.isTargetMachO()) || in LowerDYNAMIC_STACKALLOC()
25931 bool Is64Bit = Subtarget.is64Bit(); in LowerDYNAMIC_STACKALLOC()
25941 const TargetFrameLowering &TFI = *Subtarget.getFrameLowering(); in LowerDYNAMIC_STACKALLOC()
25985 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo(); in LowerDYNAMIC_STACKALLOC()
26014 if (!Subtarget.is64Bit() || in LowerVASTART()
26015 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv())) { in LowerVASTART()
26054 Subtarget.isTarget64BitLP64() ? 8 : 4, DL)); in LowerVASTART()
26058 MachinePointerInfo(SV, Subtarget.isTarget64BitLP64() ? 16 : 12)); in LowerVASTART()
26064 assert(Subtarget.is64Bit() && in LowerVAARG()
26069 if (Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv())) in LowerVAARG()
26098 assert(!Subtarget.useSoftFloat() && in LowerVAARG()
26100 Subtarget.hasSSE1()); in LowerVAARG()
26111 Subtarget.isTarget64BitLP64() ? X86ISD::VAARG_64 : X86ISD::VAARG_X32, dl, in LowerVAARG()
26121 static SDValue LowerVACOPY(SDValue Op, const X86Subtarget &Subtarget, in LowerVACOPY() argument
26125 assert(Subtarget.is64Bit() && "This code only handles 64-bit va_copy!"); in LowerVACOPY()
26126 if (Subtarget.isCallingConvWin64( in LowerVACOPY()
26140 DAG.getIntPtrConstant(Subtarget.isTarget64BitLP64() ? 24 : 16, DL), in LowerVACOPY()
26141 Align(Subtarget.isTarget64BitLP64() ? 8 : 4), /*isVolatile*/ false, false, in LowerVACOPY()
26220 const X86Subtarget &Subtarget, in getTargetVShiftNode() argument
26287 } else if (Subtarget.hasSSE41()) { in getTargetVShiftNode()
26316 const X86Subtarget &Subtarget, SelectionDAG &DAG, in getMaskNode() argument
26326 if (Mask.getSimpleValueType() == MVT::i64 && Subtarget.is32Bit()) { in getMaskNode()
26328 assert(Subtarget.hasBWI() && "Expected AVX512BW target!"); in getMaskNode()
26356 const X86Subtarget &Subtarget, in getVectorMaskingNode() argument
26366 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in getVectorMaskingNode()
26369 PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl); in getVectorMaskingNode()
26382 const X86Subtarget &Subtarget, in getScalarMaskingNode() argument
26402 PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl); in getScalarMaskingNode()
26454 const X86Subtarget &Subtarget = DAG.getSubtarget<X86Subtarget>(); in recoverFramePointer() local
26455 if (Subtarget.is64Bit()) in recoverFramePointer()
26638 Mask, PassThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
26644 Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
26661 Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
26681 Mask, passThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
26687 Mask, passThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
26703 Mask, passThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
26722 return getScalarMaskingNode(NewOp, Mask, passThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
26739 Mask, passThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
26758 return getVectorMaskingNode(NewOp, Mask, PassThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
26776 Mask, PassThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
26794 Mask, PassThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
26812 Mask, PassThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
26842 PassThru = getZeroVector(VT, Subtarget, DAG, dl); in LowerINTRINSIC_WO_CHAIN()
26859 return getVectorMaskingNode(NewOp, Mask, PassThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
26872 Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
26919 Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
27001 SrcOp, ShAmt, 0, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
27026 : getZeroVector(VT, Subtarget, DAG, dl); in LowerINTRINSIC_WO_CHAIN()
27040 return getVectorMaskingNode(FixupImm, Mask, Passthru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
27042 return getScalarMaskingNode(FixupImm, Mask, Passthru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
27107 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in LowerINTRINSIC_WO_CHAIN()
27122 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in LowerINTRINSIC_WO_CHAIN()
27368 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo(); in LowerINTRINSIC_WO_CHAIN()
27464 if (Subtarget.isTargetELF()) { in LowerINTRINSIC_WO_CHAIN()
27469 *DAG.getContext(), Subtarget.is64Bit() ? X86AS::FS : X86AS::GS)); in LowerINTRINSIC_WO_CHAIN()
27482 const X86Subtarget &Subtarget) { in getAVX2GatherNode() argument
27497 Src = getZeroVector(Op.getSimpleValueType(), Subtarget, DAG, dl); in getAVX2GatherNode()
27514 const X86Subtarget &Subtarget) { in getGatherNode() argument
27531 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in getGatherNode()
27538 Src = getZeroVector(Op.getSimpleValueType(), Subtarget, DAG, dl); in getGatherNode()
27552 const X86Subtarget &Subtarget) { in getScatterNode() argument
27568 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in getScatterNode()
27583 const X86Subtarget &Subtarget) { in getPrefetchNode() argument
27596 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in getPrefetchNode()
27614 const X86Subtarget &Subtarget, in expandIntrinsicWChainHelper() argument
27633 if (Subtarget.is64Bit()) { in expandIntrinsicWChainHelper()
27645 if (Subtarget.is64Bit()) { in expandIntrinsicWChainHelper()
27667 const X86Subtarget &Subtarget, in getReadTimeStampCounter() argument
27673 /* NoRegister */0, Subtarget, in getReadTimeStampCounter()
27686 static SDValue LowerREADCYCLECOUNTER(SDValue Op, const X86Subtarget &Subtarget, in LowerREADCYCLECOUNTER() argument
27690 getReadTimeStampCounter(Op.getNode(), DL, X86::RDTSC, DAG, Subtarget, in LowerREADCYCLECOUNTER()
27754 static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget, in LowerINTRINSIC_W_CHAIN() argument
27765 if (Subtarget.is64Bit()) { in LowerINTRINSIC_W_CHAIN()
28027 Scale, Chain, Subtarget); in LowerINTRINSIC_W_CHAIN()
28038 Chain, Subtarget); in LowerINTRINSIC_W_CHAIN()
28049 Scale, Chain, Subtarget); in LowerINTRINSIC_W_CHAIN()
28062 Subtarget); in LowerINTRINSIC_W_CHAIN()
28067 getReadTimeStampCounter(Op.getNode(), dl, IntrData->Opc0, DAG, Subtarget, in LowerINTRINSIC_W_CHAIN()
28084 Subtarget, Results); in LowerINTRINSIC_W_CHAIN()
28118 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in LowerINTRINSIC_W_CHAIN()
28133 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in LowerINTRINSIC_W_CHAIN()
28159 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo(); in LowerRETURNADDR()
28182 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo(); in LowerFRAMEADDR()
28220 const TargetFrameLowering &TFI = *Subtarget.getFrameLowering(); in getRegisterByName()
28235 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo(); in getRegisterByName()
28251 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo(); in LowerFRAME_TO_ARGS_OFFSET()
28258 return Subtarget.isTarget64BitLP64() ? X86::RDX : X86::EDX; in getExceptionPointerRegister()
28260 return Subtarget.isTarget64BitLP64() ? X86::RAX : X86::EAX; in getExceptionPointerRegister()
28268 return Subtarget.isTarget64BitLP64() ? X86::RDX : X86::EDX; in getExceptionSelectorRegister()
28272 return Subtarget.isTargetWin64(); in needsFixedCatchObjects()
28282 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo(); in LowerEH_RETURN()
28310 if (!Subtarget.is64Bit()) { in lowerEH_SJLJ_SETJMP()
28311 const X86InstrInfo *TII = Subtarget.getInstrInfo(); in lowerEH_SJLJ_SETJMP()
28346 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); in LowerINIT_TRAMPOLINE()
28348 if (Subtarget.is64Bit()) { in LowerINIT_TRAMPOLINE()
28626 if (Subtarget.hasSSE1()) { in LowerSET_ROUNDING()
28665 const X86Subtarget &Subtarget) { in LowerVectorCTLZ_AVX512CDI() argument
28677 (NumElems == 16 && !Subtarget.canExtendTo512DQ())) in LowerVectorCTLZ_AVX512CDI()
28695 const X86Subtarget &Subtarget, in LowerVectorCTLZInRegLUT() argument
28777 const X86Subtarget &Subtarget, in LowerVectorCTLZ() argument
28781 if (Subtarget.hasCDI() && in LowerVectorCTLZ()
28783 (Subtarget.canExtendTo512DQ() || VT.getVectorElementType() != MVT::i8)) in LowerVectorCTLZ()
28784 return LowerVectorCTLZ_AVX512CDI(Op, DAG, Subtarget); in LowerVectorCTLZ()
28787 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerVectorCTLZ()
28791 if (VT.is512BitVector() && !Subtarget.hasBWI()) in LowerVectorCTLZ()
28794 assert(Subtarget.hasSSSE3() && "Expected SSSE3 support for PSHUFB"); in LowerVectorCTLZ()
28795 return LowerVectorCTLZInRegLUT(Op, DL, Subtarget, DAG); in LowerVectorCTLZ()
28798 static SDValue LowerCTLZ(SDValue Op, const X86Subtarget &Subtarget, in LowerCTLZ() argument
28807 return LowerVectorCTLZ(Op, dl, Subtarget, DAG); in LowerCTLZ()
28837 static SDValue LowerCTTZ(SDValue Op, const X86Subtarget &Subtarget, in LowerCTTZ() argument
28859 const X86Subtarget &Subtarget) { in lowerAddSub() argument
28862 return lowerAddSubToHorizontalOp(Op, DAG, Subtarget); in lowerAddSub()
28874 const X86Subtarget &Subtarget) { in LowerADDSAT_SUBSAT() argument
28881 (VT.is256BitVector() && !Subtarget.hasInt256())) { in LowerADDSAT_SUBSAT()
28894 if (!TLI.isOperationLegal(ISD::UMAX, VT) || useVPTERNLOG(Subtarget, VT)) { in LowerADDSAT_SUBSAT()
28943 static SDValue LowerABS(SDValue Op, const X86Subtarget &Subtarget, in LowerABS() argument
28959 if ((VT == MVT::v2i64 || VT == MVT::v4i64) && Subtarget.hasSSE41()) { in LowerABS()
28967 if (VT.is256BitVector() && !Subtarget.hasInt256()) { in LowerABS()
28973 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) in LowerABS()
28980 static SDValue LowerAVG(SDValue Op, const X86Subtarget &Subtarget, in LowerAVG() argument
28985 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerAVG()
28995 static SDValue LowerMINMAX(SDValue Op, const X86Subtarget &Subtarget, in LowerMINMAX() argument
29000 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerMINMAX()
29010 static SDValue LowerMUL(SDValue Op, const X86Subtarget &Subtarget, in LowerMUL() argument
29016 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerMUL()
29019 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) in LowerMUL()
29030 if ((VT == MVT::v16i8 && Subtarget.hasInt256()) || in LowerMUL()
29031 (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) { in LowerMUL()
29073 return getPack(DAG, Subtarget, dl, VT, RLo, RHi); in LowerMUL()
29078 assert(Subtarget.hasSSE2() && !Subtarget.hasSSE41() && in LowerMUL()
29106 assert(!Subtarget.hasDQI() && "DQI should use MULLQ"); in LowerMUL()
29155 const X86Subtarget &Subtarget, in LowervXi8MulWithUNPCK() argument
29227 *Low = getPack(DAG, Subtarget, dl, VT, RLo, RHi); in LowervXi8MulWithUNPCK()
29229 return getPack(DAG, Subtarget, dl, VT, RLo, RHi, /*PackHiHalf*/ true); in LowervXi8MulWithUNPCK()
29232 static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget, in LowerMULH() argument
29242 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerMULH()
29245 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) in LowerMULH()
29249 assert((VT == MVT::v4i32 && Subtarget.hasSSE2()) || in LowerMULH()
29250 (VT == MVT::v8i32 && Subtarget.hasInt256()) || in LowerMULH()
29251 (VT == MVT::v16i32 && Subtarget.hasAVX512())); in LowerMULH()
29278 (IsSigned && Subtarget.hasSSE41()) ? X86ISD::PMULDQ : X86ISD::PMULUDQ; in LowerMULH()
29299 if (IsSigned && !Subtarget.hasSSE41()) { in LowerMULH()
29314 assert((VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) || in LowerMULH()
29315 (VT == MVT::v64i8 && Subtarget.hasBWI())) && in LowerMULH()
29324 if ((VT == MVT::v16i8 && Subtarget.hasInt256()) || in LowerMULH()
29325 (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) { in LowerMULH()
29335 return LowervXi8MulWithUNPCK(A, B, dl, VT, IsSigned, Subtarget, DAG); in LowerMULH()
29339 static SDValue LowerMULO(SDValue Op, const X86Subtarget &Subtarget, in LowerMULO() argument
29353 if ((VT == MVT::v32i8 && !Subtarget.hasInt256()) || in LowerMULO()
29354 (VT == MVT::v64i8 && !Subtarget.hasBWI())) { in LowerMULO()
29384 if ((VT == MVT::v16i8 && Subtarget.hasInt256()) || in LowerMULO()
29385 (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) { in LowerMULO()
29399 (Subtarget.hasBWI() || Subtarget.canExtendTo512DQ())) { in LowerMULO()
29409 if (!Subtarget.hasBWI()) { in LowerMULO()
29427 (Subtarget.hasBWI() || Subtarget.canExtendTo512DQ())) { in LowerMULO()
29430 if (!Subtarget.hasBWI()) { in LowerMULO()
29451 LowervXi8MulWithUNPCK(A, B, dl, VT, IsSigned, Subtarget, DAG, &Low); in LowerMULO()
29471 assert(Subtarget.isTargetWin64() && "Unexpected target"); in LowerWin64_i128OP()
29530 assert(Subtarget.isTargetWin64() && "Unexpected target"); in LowerWin64_FP_TO_INT128()
29563 assert(Subtarget.isTargetWin64() && "Unexpected target"); in LowerWin64_INT128_TO_FP()
29600 static bool supportedVectorShiftWithImm(MVT VT, const X86Subtarget &Subtarget, in supportedVectorShiftWithImm() argument
29608 if (VT.is512BitVector() && Subtarget.useAVX512Regs() && in supportedVectorShiftWithImm()
29609 (VT.getScalarSizeInBits() > 16 || Subtarget.hasBWI())) in supportedVectorShiftWithImm()
29612 bool LShift = (VT.is128BitVector() && Subtarget.hasSSE2()) || in supportedVectorShiftWithImm()
29613 (VT.is256BitVector() && Subtarget.hasInt256()); in supportedVectorShiftWithImm()
29615 bool AShift = LShift && (Subtarget.hasAVX512() || in supportedVectorShiftWithImm()
29623 bool supportedVectorShiftWithBaseAmnt(MVT VT, const X86Subtarget &Subtarget, in supportedVectorShiftWithBaseAmnt() argument
29625 return supportedVectorShiftWithImm(VT, Subtarget, Opcode); in supportedVectorShiftWithBaseAmnt()
29630 static bool supportedVectorVarShift(MVT VT, const X86Subtarget &Subtarget, in supportedVectorVarShift() argument
29635 if (!Subtarget.hasInt256() || VT.getScalarSizeInBits() < 16) in supportedVectorVarShift()
29639 if (VT.getScalarSizeInBits() == 16 && !Subtarget.hasBWI()) in supportedVectorVarShift()
29642 if (Subtarget.hasAVX512() && in supportedVectorVarShift()
29643 (Subtarget.useAVX512Regs() || !VT.is512BitVector())) in supportedVectorVarShift()
29652 const X86Subtarget &Subtarget) { in LowerShiftByScalarImmediate() argument
29665 if (ShiftAmt == 63 && Subtarget.hasSSE42()) { in LowerShiftByScalarImmediate()
29666 assert((VT != MVT::v4i64 || Subtarget.hasInt256()) && in LowerShiftByScalarImmediate()
29709 if (supportedVectorShiftWithImm(VT, Subtarget, Op.getOpcode())) in LowerShiftByScalarImmediate()
29713 if (((!Subtarget.hasXOP() && VT == MVT::v2i64) || in LowerShiftByScalarImmediate()
29714 (Subtarget.hasInt256() && VT == MVT::v4i64)) && in LowerShiftByScalarImmediate()
29718 if (VT == MVT::v16i8 || (Subtarget.hasInt256() && VT == MVT::v32i8) || in LowerShiftByScalarImmediate()
29719 (Subtarget.hasBWI() && VT == MVT::v64i8)) { in LowerShiftByScalarImmediate()
29746 if (VT == MVT::v16i8 && Subtarget.hasXOP()) in LowerShiftByScalarImmediate()
29783 const X86Subtarget &Subtarget) { in LowerShiftByScalarVariable() argument
29793 if (supportedVectorShiftWithBaseAmnt(VT, Subtarget, Opcode)) in LowerShiftByScalarVariable()
29795 Subtarget, DAG); in LowerShiftByScalarVariable()
29798 if (((VT == MVT::v16i8 && !Subtarget.canExtendTo512DQ()) || in LowerShiftByScalarVariable()
29799 (VT == MVT::v32i8 && !Subtarget.canExtendTo512BW()) || in LowerShiftByScalarVariable()
29801 !Subtarget.hasXOP()) { in LowerShiftByScalarVariable()
29804 if (supportedVectorShiftWithBaseAmnt(ExtVT, Subtarget, Opcode)) { in LowerShiftByScalarVariable()
29812 BaseShAmt, BaseShAmtIdx, Subtarget, DAG); in LowerShiftByScalarVariable()
29822 BaseShAmtIdx, Subtarget, DAG); in LowerShiftByScalarVariable()
29832 BaseShAmtIdx, Subtarget, DAG); in LowerShiftByScalarVariable()
29847 const X86Subtarget &Subtarget, in convertShiftLeftToScale() argument
29851 (Subtarget.hasInt256() && VT == MVT::v16i16) || in convertShiftLeftToScale()
29852 (Subtarget.hasAVX512() && VT == MVT::v32i16) || in convertShiftLeftToScale()
29853 (!Subtarget.hasAVX512() && VT == MVT::v16i8) || in convertShiftLeftToScale()
29854 (Subtarget.hasInt256() && VT == MVT::v32i8) || in convertShiftLeftToScale()
29855 (Subtarget.hasBWI() && VT == MVT::v64i8))) in convertShiftLeftToScale()
29887 if (VT == MVT::v8i16 && !Subtarget.hasAVX2()) { in convertShiftLeftToScale()
29891 Lo = convertShiftLeftToScale(Lo, dl, Subtarget, DAG); in convertShiftLeftToScale()
29892 Hi = convertShiftLeftToScale(Hi, dl, Subtarget, DAG); in convertShiftLeftToScale()
29893 if (Subtarget.hasSSE41()) in convertShiftLeftToScale()
29895 return getPack(DAG, Subtarget, dl, VT, Lo, Hi); in convertShiftLeftToScale()
29901 static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget, in LowerShift() argument
29915 assert(Subtarget.hasSSE2() && "Only custom lower when we have SSE2!"); in LowerShift()
29917 if (SDValue V = LowerShiftByScalarImmediate(Op, DAG, Subtarget)) in LowerShift()
29920 if (SDValue V = LowerShiftByScalarVariable(Op, DAG, Subtarget)) in LowerShift()
29923 if (supportedVectorVarShift(VT, Subtarget, Opc)) in LowerShift()
29929 if (((VT == MVT::v2i64 && !Subtarget.hasXOP()) || in LowerShift()
29930 (VT == MVT::v4i64 && Subtarget.hasInt256())) && in LowerShift()
29942 if (Subtarget.hasXOP() && (VT == MVT::v2i64 || VT == MVT::v4i32 || in LowerShift()
29976 (VT == MVT::v16i16 && Subtarget.hasInt256()))) { in LowerShift()
30003 (VT == MVT::v4i32 || Subtarget.hasSSE41() || Opc != ISD::SHL || in LowerShift()
30021 if (Opc == ISD::SHL && !(VT == MVT::v32i8 && (Subtarget.hasXOP() || in LowerShift()
30022 Subtarget.canExtendTo512BW()))) in LowerShift()
30023 if (SDValue Scale = convertShiftLeftToScale(Amt, dl, Subtarget, DAG)) in LowerShift()
30029 (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256()))) { in LowerShift()
30032 if (SDValue Scale = convertShiftLeftToScale(RAmt, dl, Subtarget, DAG)) { in LowerShift()
30045 (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256())) && in LowerShift()
30046 ((Subtarget.hasSSE41() && !Subtarget.hasXOP() && in LowerShift()
30047 !Subtarget.hasAVX512()) || in LowerShift()
30051 if (SDValue Scale = convertShiftLeftToScale(RAmt, dl, Subtarget, DAG)) { in LowerShift()
30081 if (Subtarget.hasAVX()) { in LowerShift()
30108 if (Subtarget.hasSSE41()) { in LowerShift()
30122 if ((Subtarget.hasInt256() && VT == MVT::v8i16) || in LowerShift()
30123 (Subtarget.canExtendTo512DQ() && VT == MVT::v16i16) || in LowerShift()
30124 (Subtarget.canExtendTo512DQ() && VT == MVT::v16i8) || in LowerShift()
30125 (Subtarget.canExtendTo512BW() && VT == MVT::v32i8) || in LowerShift()
30126 (Subtarget.hasBWI() && Subtarget.hasVLX() && VT == MVT::v16i8)) { in LowerShift()
30127 assert((!Subtarget.hasBWI() || VT == MVT::v32i8 || VT == MVT::v16i8) && in LowerShift()
30129 MVT EvtSVT = Subtarget.hasBWI() ? MVT::i16 : MVT::i32; in LowerShift()
30141 (VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) || in LowerShift()
30142 (VT == MVT::v64i8 && Subtarget.hasBWI())) && in LowerShift()
30143 !Subtarget.hasXOP()) { in LowerShift()
30156 if (VT == MVT::v16i8 && Subtarget.hasInt256()) { in LowerShift()
30188 (VT == MVT::v32i8 && Subtarget.hasInt256() && !Subtarget.hasXOP()) || in LowerShift()
30189 (VT == MVT::v64i8 && Subtarget.hasBWI())) { in LowerShift()
30204 } else if (Subtarget.hasSSE41()) { in LowerShift()
30296 if (Subtarget.hasInt256() && !Subtarget.hasXOP() && VT == MVT::v16i16) { in LowerShift()
30317 bool UseSSE41 = Subtarget.hasSSE41() && in LowerShift()
30388 static SDValue LowerFunnelShift(SDValue Op, const X86Subtarget &Subtarget, in LowerFunnelShift() argument
30405 if (Subtarget.hasVBMI2() && EltSizeInBits > 8) { in LowerFunnelShift()
30413 {Op0, Op1, Imm}, DAG, Subtarget); in LowerFunnelShift()
30416 {Op0, Op1, Amt}, DAG, Subtarget); in LowerFunnelShift()
30443 if ((VT.is256BitVector() && ((Subtarget.hasXOP() && EltSizeInBits < 16) || in LowerFunnelShift()
30444 !Subtarget.hasAVX2())) || in LowerFunnelShift()
30445 (VT.is512BitVector() && !Subtarget.useBWIRegs() && in LowerFunnelShift()
30453 if (supportedVectorShiftWithBaseAmnt(ExtVT, Subtarget, ShiftOpc)) { in LowerFunnelShift()
30463 ScalarAmtIdx, Subtarget, DAG); in LowerFunnelShift()
30465 ScalarAmtIdx, Subtarget, DAG); in LowerFunnelShift()
30466 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, !IsFSHR); in LowerFunnelShift()
30471 std::min<unsigned>(EltSizeInBits * 2, Subtarget.hasBWI() ? 16 : 32)); in LowerFunnelShift()
30475 if (supportedVectorVarShift(VT, Subtarget, ShiftOpc) || Subtarget.hasXOP()) in LowerFunnelShift()
30481 if (supportedVectorVarShift(WideVT, Subtarget, ShiftOpc) && in LowerFunnelShift()
30482 supportedVectorShiftWithImm(WideVT, Subtarget, ShiftOpc)) { in LowerFunnelShift()
30497 if (((IsCst || !Subtarget.hasAVX512()) && !IsFSHR && EltSizeInBits <= 16) || in LowerFunnelShift()
30498 supportedVectorVarShift(ExtVT, Subtarget, ShiftOpc)) { in LowerFunnelShift()
30506 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, !IsFSHR); in LowerFunnelShift()
30518 bool ExpandFunnel = !OptForSize && Subtarget.isSHLDSlow(); in LowerFunnelShift()
30554 static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget, in LowerRotate() argument
30576 if (Subtarget.hasAVX512() && 32 <= EltSizeInBits) { in LowerRotate()
30590 if (Subtarget.hasVBMI2() && 16 == EltSizeInBits) { in LowerRotate()
30604 if (Subtarget.hasXOP()) in LowerRotate()
30610 if (VT.is256BitVector() && (Subtarget.hasXOP() || !Subtarget.hasAVX2())) in LowerRotate()
30616 if (Subtarget.hasXOP()) { in LowerRotate()
30636 if (VT.is512BitVector() && !Subtarget.useBWIRegs()) in LowerRotate()
30642 Subtarget.hasAVX2()) || in LowerRotate()
30643 ((VT == MVT::v32i16 || VT == MVT::v64i8) && Subtarget.useBWIRegs())) && in LowerRotate()
30658 if (EltSizeInBits == 16 && Subtarget.hasSSE41()) { in LowerRotate()
30666 BaseRotAmtIdx, Subtarget, DAG); in LowerRotate()
30668 BaseRotAmtIdx, Subtarget, DAG); in LowerRotate()
30669 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, IsROTL); in LowerRotate()
30679 MVT::getVectorVT(Subtarget.hasBWI() ? MVT::i16 : MVT::i32, NumElts); in LowerRotate()
30685 if (supportedVectorVarShift(WideVT, Subtarget, ShiftOpc) && in LowerRotate()
30686 supportedVectorShiftWithImm(WideVT, Subtarget, ShiftOpc)) { in LowerRotate()
30705 if (IsConstAmt || supportedVectorVarShift(ExtVT, Subtarget, ShiftOpc)) { in LowerRotate()
30713 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, IsROTL); in LowerRotate()
30719 if (Subtarget.hasSSE41()) { in LowerRotate()
30737 if (!IsROTL && !useVPTERNLOG(Subtarget, VT)) { in LowerRotate()
30783 bool LegalVarShifts = supportedVectorVarShift(VT, Subtarget, ISD::SHL) && in LowerRotate()
30784 supportedVectorVarShift(VT, Subtarget, ISD::SRL); in LowerRotate()
30788 if (IsSplatAmt || LegalVarShifts || (Subtarget.hasAVX2() && !ConstantAmt)) { in LowerRotate()
30810 SDValue Scale = convertShiftLeftToScale(Amt, DL, Subtarget, DAG); in LowerRotate()
30851 return Subtarget.canUseCMPXCHG8B() && !Subtarget.is64Bit(); in needsCmpXchgNb()
30853 return Subtarget.canUseCMPXCHG16B(); in needsCmpXchgNb()
30864 if (MemType->getPrimitiveSizeInBits() == 64 && !Subtarget.is64Bit() && in shouldExpandAtomicStoreInIR()
30865 !Subtarget.useSoftFloat() && !NoImplicitFloatOps && in shouldExpandAtomicStoreInIR()
30866 (Subtarget.hasSSE1() || Subtarget.hasX87())) in shouldExpandAtomicStoreInIR()
30884 if (MemType->getPrimitiveSizeInBits() == 64 && !Subtarget.is64Bit() && in shouldExpandAtomicLoadInIR()
30885 !Subtarget.useSoftFloat() && !NoImplicitFloatOps && in shouldExpandAtomicLoadInIR()
30886 (Subtarget.hasSSE1() || Subtarget.hasX87())) in shouldExpandAtomicLoadInIR()
30954 unsigned NativeWidth = Subtarget.is64Bit() ? 64 : 32; in shouldExpandAtomicRMWInIR()
30994 unsigned NativeWidth = Subtarget.is64Bit() ? 64 : 32; in lowerIdempotentRMWIntoFencedLoad()
31038 if (!Subtarget.hasMFence()) in lowerIdempotentRMWIntoFencedLoad()
31075 const X86Subtarget &Subtarget, SDValue Chain, in emitLockedStackOp() argument
31102 auto &TFL = *Subtarget.getFrameLowering(); in emitLockedStackOp()
31105 if (Subtarget.is64Bit()) { in emitLockedStackOp()
31135 static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget &Subtarget, in LowerATOMIC_FENCE() argument
31147 if (Subtarget.hasMFence()) in LowerATOMIC_FENCE()
31151 return emitLockedStackOp(DAG, Subtarget, Chain, dl); in LowerATOMIC_FENCE()
31158 static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget &Subtarget, in LowerCMP_SWAP() argument
31170 assert(Subtarget.is64Bit() && "Node not type legal!"); in LowerCMP_SWAP()
31198 const X86Subtarget &Subtarget) { in getPMOVMSKB() argument
31204 Lo = getPMOVMSKB(DL, Lo, DAG, Subtarget); in getPMOVMSKB()
31205 Hi = getPMOVMSKB(DL, Hi, DAG, Subtarget); in getPMOVMSKB()
31212 if (InVT == MVT::v32i8 && !Subtarget.hasInt256()) { in getPMOVMSKB()
31225 static SDValue LowerBITCAST(SDValue Op, const X86Subtarget &Subtarget, in LowerBITCAST() argument
31234 assert(!Subtarget.is64Bit() && "Expected 32-bit mode"); in LowerBITCAST()
31235 assert(Subtarget.hasBWI() && "Expected BWI target"); in LowerBITCAST()
31248 assert(!Subtarget.hasAVX512() && "Should use K-registers with AVX512"); in LowerBITCAST()
31252 V = getPMOVMSKB(DL, V, DAG, Subtarget); in LowerBITCAST()
31259 assert(Subtarget.hasSSE2() && "Requires at least SSE2!"); in LowerBITCAST()
31274 assert(SrcVT == MVT::i64 && !Subtarget.is64Bit() && in LowerBITCAST()
31296 const X86Subtarget &Subtarget, in LowerHorizontalByteSum() argument
31360 const X86Subtarget &Subtarget, in LowerVectorCTPOPInRegLUT() argument
31406 static SDValue LowerVectorCTPOP(SDValue Op, const X86Subtarget &Subtarget, in LowerVectorCTPOP() argument
31415 if (Subtarget.hasVPOPCNTDQ()) { in LowerVectorCTPOP()
31419 if (NumElems < 16 || (NumElems == 16 && Subtarget.canExtendTo512DQ())) { in LowerVectorCTPOP()
31428 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerVectorCTPOP()
31432 if (VT.is512BitVector() && !Subtarget.hasBWI()) in LowerVectorCTPOP()
31440 return LowerHorizontalByteSum(PopCnt8, VT, Subtarget, DAG); in LowerVectorCTPOP()
31444 if (!Subtarget.hasSSSE3()) in LowerVectorCTPOP()
31447 return LowerVectorCTPOPInRegLUT(Op0, DL, Subtarget, DAG); in LowerVectorCTPOP()
31450 static SDValue LowerCTPOP(SDValue Op, const X86Subtarget &Subtarget, in LowerCTPOP() argument
31454 return LowerVectorCTPOP(Op, Subtarget, DAG); in LowerCTPOP()
31502 static SDValue LowerBITREVERSE(SDValue Op, const X86Subtarget &Subtarget, in LowerBITREVERSE() argument
31506 if (Subtarget.hasXOP() && !VT.is512BitVector()) in LowerBITREVERSE()
31509 assert(Subtarget.hasSSSE3() && "SSSE3 required for BITREVERSE"); in LowerBITREVERSE()
31518 if (VT == MVT::v64i8 && !Subtarget.hasBWI()) in LowerBITREVERSE()
31522 if (VT == MVT::v32i8 && !Subtarget.hasInt256()) in LowerBITREVERSE()
31528 if (Subtarget.hasGFNI()) { in LowerBITREVERSE()
31567 static SDValue LowerPARITY(SDValue Op, const X86Subtarget &Subtarget, in LowerPARITY() argument
31586 if (Subtarget.hasPOPCNT()) in LowerPARITY()
31624 const X86Subtarget &Subtarget) { in lowerAtomicArithWithLOCK() argument
31656 const X86Subtarget &Subtarget) { in lowerAtomicArith() argument
31697 SDValue NewChain = emitLockedStackOp(DAG, Subtarget, Chain, DL); in lowerAtomicArith()
31711 SDValue LockOp = lowerAtomicArithWithLOCK(N, DAG, Subtarget); in lowerAtomicArith()
31720 const X86Subtarget &Subtarget) { in LowerATOMIC_STORE() argument
31740 if (!Subtarget.useSoftFloat() && !NoImplicitFloatOps) { in LowerATOMIC_STORE()
31742 if (Subtarget.hasSSE1()) { in LowerATOMIC_STORE()
31745 MVT StVT = Subtarget.hasSSE2() ? MVT::v2i64 : MVT::v4f32; in LowerATOMIC_STORE()
31751 } else if (Subtarget.hasX87()) { in LowerATOMIC_STORE()
31779 Chain = emitLockedStackOp(DAG, Subtarget, Chain, dl); in LowerATOMIC_STORE()
31829 static SDValue LowerFSINCOS(SDValue Op, const X86Subtarget &Subtarget, in LowerFSINCOS() argument
31831 assert(Subtarget.isTargetDarwin() && Subtarget.is64Bit()); in LowerFSINCOS()
31934 static SDValue LowerMSCATTER(SDValue Op, const X86Subtarget &Subtarget, in LowerMSCATTER() argument
31936 assert(Subtarget.hasAVX512() && in LowerMSCATTER()
31954 if (Index.getValueType() == MVT::v2i64 && Subtarget.hasVLX()) { in LowerMSCATTER()
31975 if (!Subtarget.hasVLX() && !VT.is512BitVector() && in LowerMSCATTER()
31997 static SDValue LowerMLOAD(SDValue Op, const X86Subtarget &Subtarget, in LowerMLOAD() argument
32016 getZeroVector(VT, Subtarget, DAG, dl), N->getMemoryVT(), in LowerMLOAD()
32024 assert((!N->isExpandingLoad() || Subtarget.hasAVX512()) && in LowerMLOAD()
32030 assert(Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() && in LowerMLOAD()
32034 (Subtarget.hasBWI() && in LowerMLOAD()
32063 static SDValue LowerMSTORE(SDValue Op, const X86Subtarget &Subtarget, in LowerMSTORE() argument
32072 assert((!N->isCompressingStore() || Subtarget.hasAVX512()) && in LowerMSTORE()
32078 assert(Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() && in LowerMSTORE()
32082 (Subtarget.hasBWI() && in LowerMSTORE()
32105 static SDValue LowerMGATHER(SDValue Op, const X86Subtarget &Subtarget, in LowerMGATHER() argument
32107 assert(Subtarget.hasAVX2() && in LowerMGATHER()
32127 if (Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() && in LowerMGATHER()
32146 PassThru = getZeroVector(VT, Subtarget, DAG, dl); in LowerMGATHER()
32216 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, Subtarget, DAG); in LowerOperation()
32218 return LowerCMP_SWAP(Op, Subtarget, DAG); in LowerOperation()
32219 case ISD::CTPOP: return LowerCTPOP(Op, Subtarget, DAG); in LowerOperation()
32224 case ISD::ATOMIC_LOAD_AND: return lowerAtomicArith(Op, DAG, Subtarget); in LowerOperation()
32225 case ISD::ATOMIC_STORE: return LowerATOMIC_STORE(Op, DAG, Subtarget); in LowerOperation()
32226 case ISD::BITREVERSE: return LowerBITREVERSE(Op, Subtarget, DAG); in LowerOperation()
32227 case ISD::PARITY: return LowerPARITY(Op, Subtarget, DAG); in LowerOperation()
32229 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, Subtarget, DAG); in LowerOperation()
32230 case ISD::VECTOR_SHUFFLE: return lowerVECTOR_SHUFFLE(Op, Subtarget, DAG); in LowerOperation()
32234 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, Subtarget,DAG); in LowerOperation()
32235 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op,Subtarget,DAG); in LowerOperation()
32236 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, Subtarget,DAG); in LowerOperation()
32246 case ISD::FSHR: return LowerFunnelShift(Op, Subtarget, DAG); in LowerOperation()
32252 case ISD::ZERO_EXTEND: return LowerZERO_EXTEND(Op, Subtarget, DAG); in LowerOperation()
32253 case ISD::SIGN_EXTEND: return LowerSIGN_EXTEND(Op, Subtarget, DAG); in LowerOperation()
32254 case ISD::ANY_EXTEND: return LowerANY_EXTEND(Op, Subtarget, DAG); in LowerOperation()
32257 return LowerEXTEND_VECTOR_INREG(Op, Subtarget, DAG); in LowerOperation()
32273 case ISD::LOAD: return LowerLoad(Op, Subtarget, DAG); in LowerOperation()
32274 case ISD::STORE: return LowerStore(Op, Subtarget, DAG); in LowerOperation()
32293 case ISD::VACOPY: return LowerVACOPY(Op, Subtarget, DAG); in LowerOperation()
32296 case ISD::INTRINSIC_W_CHAIN: return LowerINTRINSIC_W_CHAIN(Op, Subtarget, DAG); in LowerOperation()
32313 case ISD::CTLZ_ZERO_UNDEF: return LowerCTLZ(Op, Subtarget, DAG); in LowerOperation()
32315 case ISD::CTTZ_ZERO_UNDEF: return LowerCTTZ(Op, Subtarget, DAG); in LowerOperation()
32316 case ISD::MUL: return LowerMUL(Op, Subtarget, DAG); in LowerOperation()
32318 case ISD::MULHU: return LowerMULH(Op, Subtarget, DAG); in LowerOperation()
32320 case ISD::ROTR: return LowerRotate(Op, Subtarget, DAG); in LowerOperation()
32323 case ISD::SHL: return LowerShift(Op, Subtarget, DAG); in LowerOperation()
32329 case ISD::UMULO: return LowerMULO(Op, Subtarget, DAG); in LowerOperation()
32330 case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, Subtarget,DAG); in LowerOperation()
32331 case ISD::BITCAST: return LowerBITCAST(Op, Subtarget, DAG); in LowerOperation()
32337 case ISD::SUB: return lowerAddSub(Op, DAG, Subtarget); in LowerOperation()
32341 case ISD::SSUBSAT: return LowerADDSAT_SUBSAT(Op, DAG, Subtarget); in LowerOperation()
32345 case ISD::UMIN: return LowerMINMAX(Op, Subtarget, DAG); in LowerOperation()
32346 case ISD::ABS: return LowerABS(Op, Subtarget, DAG); in LowerOperation()
32347 case ISD::AVGCEILU: return LowerAVG(Op, Subtarget, DAG); in LowerOperation()
32348 case ISD::FSINCOS: return LowerFSINCOS(Op, Subtarget, DAG); in LowerOperation()
32349 case ISD::MLOAD: return LowerMLOAD(Op, Subtarget, DAG); in LowerOperation()
32350 case ISD::MSTORE: return LowerMSTORE(Op, Subtarget, DAG); in LowerOperation()
32351 case ISD::MGATHER: return LowerMGATHER(Op, Subtarget, DAG); in LowerOperation()
32352 case ISD::MSCATTER: return LowerMSCATTER(Op, Subtarget, DAG); in LowerOperation()
32482 assert(Subtarget.hasSSE2() && "Requires at least SSE2!"); in ReplaceNodeResults()
32588 if (Subtarget.hasAVX512() && isTypeLegal(InVT)) { in ReplaceNodeResults()
32590 if ((InBits == 256 && Subtarget.hasVLX()) || InBits == 512) { in ReplaceNodeResults()
32602 if (Subtarget.hasVLX() && InVT == MVT::v8i64 && VT == MVT::v8i8 && in ReplaceNodeResults()
32632 if (!Subtarget.hasSSE41() && VT == MVT::v4i64 && in ReplaceNodeResults()
32731 if (VT.isVector() && Subtarget.hasFP16() && in ReplaceNodeResults()
32824 assert((!IsStrict || IsSigned || Subtarget.hasAVX512()) && in ReplaceNodeResults()
32826 assert(Subtarget.hasSSE2() && "Requires at least SSE2!"); in ReplaceNodeResults()
32830 if (!IsSigned && !Subtarget.hasAVX512()) { in ReplaceNodeResults()
32832 expandFP_TO_UINT_SSE(MVT::v4i32, Src, dl, DAG, Subtarget); in ReplaceNodeResults()
32844 if (!IsSigned && !Subtarget.hasVLX()) { in ReplaceNodeResults()
32890 if ((Subtarget.hasDQI() && VT == MVT::i64 && in ReplaceNodeResults()
32892 (Subtarget.hasFP16() && SrcVT == MVT::f16)) { in ReplaceNodeResults()
32893 assert(!Subtarget.is64Bit() && "i64 should be legal"); in ReplaceNodeResults()
32894 unsigned NumElts = Subtarget.hasVLX() ? 2 : 8; in ReplaceNodeResults()
32926 if (VT == MVT::i128 && Subtarget.isTargetWin64()) { in ReplaceNodeResults()
32958 if (VT.getVectorElementType() == MVT::f16 && Subtarget.hasFP16() && in ReplaceNodeResults()
32959 Subtarget.hasVLX()) { in ReplaceNodeResults()
32983 if (Subtarget.hasDQI() && Subtarget.hasVLX() && SrcVT == MVT::v2i64) { in ReplaceNodeResults()
32997 if (SrcVT == MVT::v2i64 && !IsSigned && Subtarget.is64Bit() && in ReplaceNodeResults()
32998 Subtarget.hasSSE41() && !Subtarget.hasAVX512()) { in ReplaceNodeResults()
33041 if (IsSigned || Subtarget.hasAVX512()) { in ReplaceNodeResults()
33056 assert(Subtarget.hasSSE2() && "Requires at least SSE2!"); in ReplaceNodeResults()
33092 if (!Subtarget.hasFP16() && VT.getVectorElementType() == MVT::f16) { in ReplaceNodeResults()
33093 assert(Subtarget.hasF16C() && "Cannot widen f16 without F16C"); in ReplaceNodeResults()
33127 if (!Subtarget.hasFP16() || !Subtarget.hasVLX()) in ReplaceNodeResults()
33150 return getReadTimeStampCounter(N, dl, X86::RDTSC, DAG, Subtarget, in ReplaceNodeResults()
33153 return getReadTimeStampCounter(N, dl, X86::RDTSCP, DAG, Subtarget, in ReplaceNodeResults()
33156 expandIntrinsicWChainHelper(N, dl, DAG, X86::RDPMC, X86::ECX, Subtarget, in ReplaceNodeResults()
33160 expandIntrinsicWChainHelper(N, dl, DAG, X86::RDPRU, X86::ECX, Subtarget, in ReplaceNodeResults()
33164 expandIntrinsicWChainHelper(N, dl, DAG, X86::XGETBV, X86::ECX, Subtarget, in ReplaceNodeResults()
33170 return getReadTimeStampCounter(N, dl, X86::RDTSC, DAG, Subtarget, Results); in ReplaceNodeResults()
33176 assert((!Regs64bit || Subtarget.canUseCMPXCHG16B()) && in ReplaceNodeResults()
33245 if (!Subtarget.useSoftFloat() && !NoImplicitFloatOps) { in ReplaceNodeResults()
33247 if (Subtarget.hasSSE1()) { in ReplaceNodeResults()
33250 MVT LdVT = Subtarget.hasSSE2() ? MVT::v2i64 : MVT::v4f32; in ReplaceNodeResults()
33255 if (Subtarget.hasSSE2()) { in ReplaceNodeResults()
33272 if (Subtarget.hasX87()) { in ReplaceNodeResults()
33325 assert(Subtarget.hasSSE2() && "Requires at least SSE2!"); in ReplaceNodeResults()
33331 if (SrcVT == MVT::v64i1 && DstVT == MVT::i64 && Subtarget.hasBWI()) { in ReplaceNodeResults()
33332 assert(!Subtarget.is64Bit() && "Expected 32-bit mode"); in ReplaceNodeResults()
33344 assert(Subtarget.hasSSE2() && "Requires SSE2"); in ReplaceNodeResults()
33360 (Subtarget.hasVLX() || !Subtarget.hasAVX512())) { in ReplaceNodeResults()
33373 if (!Subtarget.hasVLX()) { in ReplaceNodeResults()
33402 if (Subtarget.hasSSE2()) { in ReplaceNodeResults()
33403 MVT LdVT = Subtarget.is64Bit() && VT.isInteger() ? MVT::i64 : MVT::f64; in ReplaceNodeResults()
33416 assert(Subtarget.hasSSE1() && "Expected SSE"); in ReplaceNodeResults()
33432 assert(Subtarget.hasXOP() && "Expected XOP"); in ReplaceNodeResults()
33435 Results.push_back(LowerBITREVERSE(SDValue(N, 0), Subtarget, DAG)); in ReplaceNodeResults()
33852 unsigned GVFlags = Subtarget.classifyGlobalReference(AM.BaseGV); in isLegalAddressingMode()
33865 Subtarget.is64Bit() && (AM.BaseOffs || AM.Scale > 1)) in isLegalAddressingMode()
33897 if (Subtarget.hasXOP() && in isVectorShiftByScalarCheap()
33903 if (Subtarget.hasAVX2() && (Bits == 32 || Bits == 64)) in isVectorShiftByScalarCheap()
33907 if (Subtarget.hasBWI() && Bits == 16) in isVectorShiftByScalarCheap()
33997 return Ty1->isIntegerTy(32) && Ty2->isIntegerTy(64) && Subtarget.is64Bit(); in isZExtFree()
34002 return VT1 == MVT::i32 && VT2 == MVT::i64 && Subtarget.is64Bit(); in isZExtFree()
34046 if (Subtarget.hasSSE41() && in shouldSinkOperands()
34051 } else if (Subtarget.hasSSE2() && in shouldSinkOperands()
34087 if (!Subtarget.is64Bit()) in shouldConvertPhiType()
34107 if (!Subtarget.hasAnyFMA()) in isFMAFasterThanFMulAndFAdd()
34117 return Subtarget.hasFP16(); in isFMAFasterThanFMulAndFAdd()
34137 if (!Subtarget.hasAVX512()) in shouldFoldSelectWithIdentityConstant()
34139 if (!Subtarget.hasVLX() && !VT.is512BitVector()) in shouldFoldSelectWithIdentityConstant()
34172 if (!Subtarget.hasAVX2()) in isVectorClearMaskLegal()
34182 if (Subtarget.useIndirectThunkBranches()) in areJTsAllowed()
34347 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); in EmitVAARGWithCustomInserter()
34463 TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64rm : X86::MOV32rm), in EmitVAARGWithCustomInserter()
34468 .addDisp(Disp, Subtarget.isTarget64BitLP64() ? 16 : 12) in EmitVAARGWithCustomInserter()
34472 if (Subtarget.isTarget64BitLP64()) { in EmitVAARGWithCustomInserter()
34519 TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64rm : X86::MOV32rm), in EmitVAARGWithCustomInserter()
34537 TII->get(Subtarget.isTarget64BitLP64() ? X86::ADD64ri32 : X86::ADD32ri), in EmitVAARGWithCustomInserter()
34544 TII->get(Subtarget.isTarget64BitLP64() ? X86::AND64ri32 : X86::AND32ri), in EmitVAARGWithCustomInserter()
34558 TII->get(Subtarget.isTarget64BitLP64() ? X86::ADD64ri32 : X86::ADD32ri), in EmitVAARGWithCustomInserter()
34565 TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64mr : X86::MOV32mr)) in EmitVAARGWithCustomInserter()
34704 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); in EmitLoweredCascadedSelect()
34796 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); in EmitLoweredCascadedSelect()
34854 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); in EmitLoweredSelect()
34945 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); in EmitLoweredSelect()
35005 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); in EmitLoweredProbedAlloca()
35006 const X86FrameLowering &TFI = *Subtarget.getFrameLowering(); in EmitLoweredProbedAlloca()
35099 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); in EmitLoweredSegAlloca()
35105 const bool Is64Bit = Subtarget.is64Bit(); in EmitLoweredSegAlloca()
35106 const bool IsLP64 = Subtarget.isTarget64BitLP64(); in EmitLoweredSegAlloca()
35141 IsLP64 || Subtarget.isTargetNaCl64() ? X86::RSP : X86::ESP; in EmitLoweredSegAlloca()
35173 Subtarget.getRegisterInfo()->getCallPreservedMask(*MF, CallingConv::C); in EmitLoweredSegAlloca()
35233 const TargetInstrInfo &TII = *Subtarget.getInstrInfo(); in EmitLoweredCatchRet()
35242 if (!Subtarget.is32Bit()) in EmitLoweredCatchRet()
35272 const TargetInstrInfo &TII = *Subtarget.getInstrInfo(); in EmitLoweredTLSAddr()
35301 const X86InstrInfo *TII = Subtarget.getInstrInfo(); in EmitLoweredTLSCall()
35304 assert(Subtarget.isTargetDarwin() && "Darwin only instr emitted?"); in EmitLoweredTLSCall()
35311 Subtarget.is64Bit() ? in EmitLoweredTLSCall()
35312 Subtarget.getRegisterInfo()->getDarwinTLSCallPreservedMask() : in EmitLoweredTLSCall()
35313 Subtarget.getRegisterInfo()->getCallPreservedMask(*F, CallingConv::C); in EmitLoweredTLSCall()
35314 if (Subtarget.is64Bit()) { in EmitLoweredTLSCall()
35370 static const char *getIndirectThunkSymbol(const X86Subtarget &Subtarget, in getIndirectThunkSymbol() argument
35372 if (Subtarget.useRetpolineExternalThunk()) { in getIndirectThunkSymbol()
35388 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"); in getIndirectThunkSymbol()
35391 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"); in getIndirectThunkSymbol()
35394 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"); in getIndirectThunkSymbol()
35397 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"); in getIndirectThunkSymbol()
35400 assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!"); in getIndirectThunkSymbol()
35406 if (Subtarget.useRetpolineIndirectCalls() || in getIndirectThunkSymbol()
35407 Subtarget.useRetpolineIndirectBranches()) { in getIndirectThunkSymbol()
35411 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"); in getIndirectThunkSymbol()
35414 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"); in getIndirectThunkSymbol()
35417 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"); in getIndirectThunkSymbol()
35420 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"); in getIndirectThunkSymbol()
35423 assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!"); in getIndirectThunkSymbol()
35429 if (Subtarget.useLVIControlFlowIntegrity()) { in getIndirectThunkSymbol()
35430 assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!"); in getIndirectThunkSymbol()
35442 const X86InstrInfo *TII = Subtarget.getInstrInfo(); in EmitLoweredIndirectThunk()
35453 if (Subtarget.is64Bit()) in EmitLoweredIndirectThunk()
35478 const char *Symbol = getIndirectThunkSymbol(Subtarget, AvailableReg); in EmitLoweredIndirectThunk()
35505 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); in emitSetJmpShadowStackFix()
35548 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); in emitEHSjLjSetJmp()
35549 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); in emitEHSjLjSetJmp()
35621 if (Subtarget.is64Bit()) { in emitEHSjLjSetJmp()
35634 .addMBB(restoreMBB, Subtarget.classifyBlockAddressReference()) in emitEHSjLjSetJmp()
35661 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo(); in emitEHSjLjSetJmp()
35680 Subtarget.isTarget64BitLP64() || Subtarget.isTargetNaCl64(); in emitEHSjLjSetJmp()
35708 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); in emitLongJmpShadowStackFix()
35889 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); in emitEHSjLjLongJmp()
35904 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo(); in emitEHSjLjLongJmp()
35974 const X86InstrInfo *TII = Subtarget.getInstrInfo(); in SetupEntryBlockForSjLj()
35993 if (Subtarget.is64Bit()) in SetupEntryBlockForSjLj()
36005 .addMBB(DispatchBB, Subtarget.classifyBlockAddressReference()) in SetupEntryBlockForSjLj()
36010 addFrameReference(MIB, FI, Subtarget.is64Bit() ? 56 : 36); in SetupEntryBlockForSjLj()
36023 const X86InstrInfo *TII = Subtarget.getInstrInfo(); in EmitSjLjDispatchBlock()
36100 Subtarget.isTarget64BitLP64() || Subtarget.isTargetNaCl64(); in EmitSjLjDispatchBlock()
36118 Subtarget.is64Bit() ? 8 : 4); in EmitSjLjDispatchBlock()
36124 if (Subtarget.is64Bit()) { in EmitSjLjDispatchBlock()
36249 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); in EmitInstrWithCustomInserter()
36415 return emitXBegin(MI, BB, Subtarget.getInstrInfo()); in EmitInstrWithCustomInserter()
36446 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo(); in EmitInstrWithCustomInserter()
36458 if (!Subtarget.is32Bit() || !TRI->hasBasePointer(*MF)) in EmitInstrWithCustomInserter()
36499 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo(); in EmitInstrWithCustomInserter()
36530 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo(); in EmitInstrWithCustomInserter()
36553 assert(Subtarget.is64Bit() && "Expected 64-bit mode!"); in EmitInstrWithCustomInserter()
36570 assert(Subtarget.is32Bit() && "preallocated only used in 32-bit"); in EmitInstrWithCustomInserter()
36585 assert(Subtarget.is32Bit() && "preallocated calls only used in 32-bit"); in EmitInstrWithCustomInserter()
37255 const X86Subtarget &Subtarget, unsigned &Shuffle, in matchUnaryShuffle() argument
37262 (MaskEltSize == 32 || (MaskEltSize == 16 && Subtarget.hasFP16()))) { in matchUnaryShuffle()
37270 SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT; in matchUnaryShuffle()
37277 if (AllowIntDomain && ((MaskVT.is128BitVector() && Subtarget.hasSSE41()) || in matchUnaryShuffle()
37278 (MaskVT.is256BitVector() && Subtarget.hasInt256()))) { in matchUnaryShuffle()
37311 if (((MaskEltSize == 32) || (MaskEltSize == 64 && Subtarget.hasSSE2()) || in matchUnaryShuffle()
37312 (MaskEltSize == 16 && Subtarget.hasFP16())) && in matchUnaryShuffle()
37319 SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT; in matchUnaryShuffle()
37326 if (MaskVT.is128BitVector() && Subtarget.hasSSE3() && AllowFloatDomain) { in matchUnaryShuffle()
37345 assert(Subtarget.hasAVX() && "AVX required for 256-bit vector shuffles"); in matchUnaryShuffle()
37366 assert(Subtarget.hasAVX512() && in matchUnaryShuffle()
37400 const X86Subtarget &Subtarget, in matchUnaryPermuteShuffle() argument
37414 if (Subtarget.hasAVX2() && MaskVT.is256BitVector()) { in matchUnaryPermuteShuffle()
37420 if (Subtarget.hasAVX512() && MaskVT.is512BitVector()) { in matchUnaryPermuteShuffle()
37429 } else if (AllowFloatDomain && Subtarget.hasAVX()) { in matchUnaryPermuteShuffle()
37449 !ContainsZeros && (AllowIntDomain || Subtarget.hasAVX())) { in matchUnaryPermuteShuffle()
37467 ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) || in matchUnaryPermuteShuffle()
37468 (MaskVT.is256BitVector() && Subtarget.hasAVX2()) || in matchUnaryPermuteShuffle()
37469 (MaskVT.is512BitVector() && Subtarget.hasBWI()))) { in matchUnaryPermuteShuffle()
37502 ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) || in matchUnaryPermuteShuffle()
37503 (MaskVT.is256BitVector() && Subtarget.hasAVX2()) || in matchUnaryPermuteShuffle()
37504 (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) { in matchUnaryPermuteShuffle()
37506 Mask, 0, Zeroable, Subtarget); in matchUnaryPermuteShuffle()
37507 if (0 < ShiftAmt && (!ShuffleVT.is512BitVector() || Subtarget.hasBWI() || in matchUnaryPermuteShuffle()
37516 ((MaskVT.is128BitVector() && Subtarget.hasXOP()) || in matchUnaryPermuteShuffle()
37517 Subtarget.hasAVX512())) { in matchUnaryPermuteShuffle()
37519 Subtarget, Mask); in matchUnaryPermuteShuffle()
37536 SelectionDAG &DAG, const X86Subtarget &Subtarget, in matchBinaryShuffle() argument
37548 Shuffle = Subtarget.hasSSE2() ? X86ISD::UNPCKL : X86ISD::MOVLHPS; in matchBinaryShuffle()
37549 SrcVT = DstVT = Subtarget.hasSSE2() ? MVT::v2f64 : MVT::v4f32; in matchBinaryShuffle()
37555 Shuffle = Subtarget.hasSSE2() ? X86ISD::UNPCKH : X86ISD::MOVHLPS; in matchBinaryShuffle()
37556 SrcVT = DstVT = Subtarget.hasSSE2() ? MVT::v2f64 : MVT::v4f32; in matchBinaryShuffle()
37560 Subtarget.hasSSE2() && (AllowFloatDomain || !Subtarget.hasSSE41())) { in matchBinaryShuffle()
37567 (AllowFloatDomain || !Subtarget.hasSSE41())) { in matchBinaryShuffle()
37574 Subtarget.hasFP16()) { in matchBinaryShuffle()
37582 if (((MaskVT == MVT::v8i16 || MaskVT == MVT::v16i8) && Subtarget.hasSSE2()) || in matchBinaryShuffle()
37583 ((MaskVT == MVT::v16i16 || MaskVT == MVT::v32i8) && Subtarget.hasInt256()) || in matchBinaryShuffle()
37584 ((MaskVT == MVT::v32i16 || MaskVT == MVT::v64i8) && Subtarget.hasBWI())) { in matchBinaryShuffle()
37586 Subtarget)) { in matchBinaryShuffle()
37593 if ((MaskVT == MVT::v4f32 && Subtarget.hasSSE1()) || in matchBinaryShuffle()
37594 (MaskVT.is128BitVector() && Subtarget.hasSSE2()) || in matchBinaryShuffle()
37595 (MaskVT.is256BitVector() && 32 <= EltSizeInBits && Subtarget.hasAVX()) || in matchBinaryShuffle()
37596 (MaskVT.is256BitVector() && Subtarget.hasAVX2()) || in matchBinaryShuffle()
37597 (MaskVT.is512BitVector() && Subtarget.hasAVX512())) { in matchBinaryShuffle()
37599 Subtarget)) { in matchBinaryShuffle()
37601 if (MaskVT.is256BitVector() && !Subtarget.hasAVX2()) in matchBinaryShuffle()
37702 const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, in matchBinaryPermuteShuffle() argument
37709 ((MaskVT.is128BitVector() && Subtarget.hasVLX()) || in matchBinaryPermuteShuffle()
37710 (MaskVT.is256BitVector() && Subtarget.hasVLX()) || in matchBinaryPermuteShuffle()
37711 (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) { in matchBinaryPermuteShuffle()
37727 if (AllowIntDomain && ((MaskVT.is128BitVector() && Subtarget.hasSSSE3()) || in matchBinaryPermuteShuffle()
37728 (MaskVT.is256BitVector() && Subtarget.hasAVX2()) || in matchBinaryPermuteShuffle()
37729 (MaskVT.is512BitVector() && Subtarget.hasBWI()))) { in matchBinaryPermuteShuffle()
37740 if ((NumMaskElts <= 8 && ((Subtarget.hasSSE41() && MaskVT.is128BitVector()) || in matchBinaryPermuteShuffle()
37741 (Subtarget.hasAVX() && MaskVT.is256BitVector()))) || in matchBinaryPermuteShuffle()
37742 (MaskVT == MVT::v16i16 && Subtarget.hasAVX2())) { in matchBinaryPermuteShuffle()
37759 V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1; in matchBinaryPermuteShuffle()
37760 V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2; in matchBinaryPermuteShuffle()
37766 V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1; in matchBinaryPermuteShuffle()
37767 V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2; in matchBinaryPermuteShuffle()
37778 if (AllowFloatDomain && EltSizeInBits == 32 && Subtarget.hasSSE41() && in matchBinaryPermuteShuffle()
37788 ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) || in matchBinaryPermuteShuffle()
37789 (MaskVT.is256BitVector() && Subtarget.hasAVX()) || in matchBinaryPermuteShuffle()
37790 (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) { in matchBinaryPermuteShuffle()
37794 V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1; in matchBinaryPermuteShuffle()
37795 V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2; in matchBinaryPermuteShuffle()
37804 ((MaskVT.is128BitVector() && Subtarget.hasSSE1()) || in matchBinaryPermuteShuffle()
37805 (MaskVT.is256BitVector() && Subtarget.hasAVX()) || in matchBinaryPermuteShuffle()
37806 (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) { in matchBinaryPermuteShuffle()
37820 return getZeroVector(MaskVT, Subtarget, DAG, DL); in matchBinaryPermuteShuffle()
37850 if (AllowFloatDomain && EltSizeInBits == 32 && Subtarget.hasSSE41() && in matchBinaryPermuteShuffle()
37865 const X86Subtarget &Subtarget);
37882 const X86Subtarget &Subtarget) { in combineX86ShuffleChain() argument
37895 Op = widenSubVector(Op, false, Subtarget, DAG, DL, VT.getSizeInBits()); in combineX86ShuffleChain()
37925 (RootVT.is256BitVector() && !Subtarget.hasAVX2()); in combineX86ShuffleChain()
37931 if (RootSizeInBits == 512 || (Subtarget.hasVLX() && RootSizeInBits >= 128)) { in combineX86ShuffleChain()
37980 return widenSubVector(Res, UseZero, Subtarget, DAG, DL, RootSizeInBits); in combineX86ShuffleChain()
38051 return widenSubVector(Res, Mask[1] == SM_SentinelZero, Subtarget, DAG, DL, in combineX86ShuffleChain()
38059 !Subtarget.hasAVX2()) { in combineX86ShuffleChain()
38075 !(Subtarget.hasAVX2() && isUndefOrInRange(Mask, 0, 2)) && in combineX86ShuffleChain()
38149 bool AllowIntDomain = (!FloatDomain || (Depth >= 3)) && Subtarget.hasSSE2() && in combineX86ShuffleChain()
38150 (!MaskVT.is256BitVector() || Subtarget.hasAVX2()); in combineX86ShuffleChain()
38160 if ((Subtarget.hasAVX2() || in combineX86ShuffleChain()
38161 (Subtarget.hasAVX() && 32 <= MaskEltSizeInBits)) && in combineX86ShuffleChain()
38166 X86::mayFoldLoad(V1.getOperand(0), Subtarget)) { in combineX86ShuffleChain()
38173 if (Subtarget.hasAVX2()) { in combineX86ShuffleChain()
38184 DAG, Subtarget, Shuffle, ShuffleSrcVT, ShuffleVT) && in combineX86ShuffleChain()
38195 AllowIntDomain, DAG, Subtarget, Shuffle, ShuffleVT, in combineX86ShuffleChain()
38212 Subtarget.hasSSE41() && in combineX86ShuffleChain()
38246 NewV2, DL, DAG, Subtarget, Shuffle, ShuffleSrcVT, in combineX86ShuffleChain()
38261 Subtarget, Shuffle, ShuffleVT, PermuteImm) && in combineX86ShuffleChain()
38277 if (Subtarget.hasSSE4A() && AllowIntDomain && RootSizeInBits == 128) { in combineX86ShuffleChain()
38303 if (AllowIntDomain && MaskEltSizeInBits < 64 && Subtarget.hasAVX512()) { in combineX86ShuffleChain()
38306 Subtarget)) { in combineX86ShuffleChain()
38316 Res = widenSubVector(Res, true, Subtarget, DAG, DL, RootSizeInBits); in combineX86ShuffleChain()
38322 ((RootVT.is256BitVector() && Subtarget.useAVX512Regs()) || in combineX86ShuffleChain()
38323 (RootVT.is128BitVector() && Subtarget.hasVLX())) && in combineX86ShuffleChain()
38324 (MaskEltSizeInBits > 8 || Subtarget.hasBWI()) && in combineX86ShuffleChain()
38351 Subtarget.hasFastVariableCrossLaneShuffle() ? 1 : 2; in combineX86ShuffleChain()
38353 Subtarget.hasFastVariablePerLaneShuffle() ? 1 : 2; in combineX86ShuffleChain()
38368 if (Subtarget.hasAVX2() && in combineX86ShuffleChain()
38376 if ((Subtarget.hasAVX512() && in combineX86ShuffleChain()
38379 (Subtarget.hasBWI() && in combineX86ShuffleChain()
38381 (Subtarget.hasVBMI() && in combineX86ShuffleChain()
38385 Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG); in combineX86ShuffleChain()
38393 ((Subtarget.hasAVX512() && in combineX86ShuffleChain()
38398 (Subtarget.hasBWI() && AllowBWIVPERMV3 && in combineX86ShuffleChain()
38400 (Subtarget.hasVBMI() && AllowBWIVPERMV3 && in combineX86ShuffleChain()
38407 V2 = getZeroVector(MaskVT, Subtarget, DAG, DL); in combineX86ShuffleChain()
38408 Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG); in combineX86ShuffleChain()
38417 Subtarget)) in combineX86ShuffleChain()
38423 ((Subtarget.hasAVX512() && in combineX86ShuffleChain()
38428 (Subtarget.hasBWI() && AllowBWIVPERMV3 && in combineX86ShuffleChain()
38430 (Subtarget.hasVBMI() && AllowBWIVPERMV3 && in combineX86ShuffleChain()
38434 Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG); in combineX86ShuffleChain()
38471 ((MaskVT == MVT::v8f32 && Subtarget.hasAVX()) || in combineX86ShuffleChain()
38472 (MaskVT == MVT::v16f32 && Subtarget.hasAVX512()))) { in combineX86ShuffleChain()
38487 if (AllowVariablePerLaneMask && Subtarget.hasXOP() && in combineX86ShuffleChain()
38526 ((RootVT.is128BitVector() && Subtarget.hasSSSE3()) || in combineX86ShuffleChain()
38527 (RootVT.is256BitVector() && Subtarget.hasAVX2()) || in combineX86ShuffleChain()
38528 (RootVT.is512BitVector() && Subtarget.hasBWI()))) { in combineX86ShuffleChain()
38557 Subtarget.hasXOP()) { in combineX86ShuffleChain()
38589 AllowVariableCrossLaneMask, AllowVariablePerLaneMask, DAG, Subtarget)) in combineX86ShuffleChain()
38595 ((Subtarget.hasAVX512() && in combineX86ShuffleChain()
38601 (Subtarget.hasBWI() && AllowBWIVPERMV3 && in combineX86ShuffleChain()
38604 (Subtarget.hasVBMI() && AllowBWIVPERMV3 && in combineX86ShuffleChain()
38609 Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG); in combineX86ShuffleChain()
38629 const X86Subtarget &Subtarget) { in combineX86ShuffleChainWithExtract() argument
38718 AllowVariablePerLaneMask, DAG, Subtarget)) { in combineX86ShuffleChainWithExtract()
38731 const X86Subtarget &Subtarget) { in canonicalizeShuffleMaskWithHorizOp() argument
38768 (isPack || shouldUseHorizontalOp(Ops.size() == 1, DAG, Subtarget))) { in canonicalizeShuffleMaskWithHorizOp()
38782 return getZeroVector(VT0.getSimpleVT(), Subtarget, DAG, DL); in canonicalizeShuffleMaskWithHorizOp()
38895 shouldUseHorizontalOp(SingleOp, DAG, Subtarget)) { in canonicalizeShuffleMaskWithHorizOp()
38902 SDValue Zero = getZeroVector(SrcVT, Subtarget, DAG, DL); in canonicalizeShuffleMaskWithHorizOp()
38922 const X86Subtarget &Subtarget) { in combineX86ShufflesConstants() argument
38990 return getZeroVector(Root.getSimpleValueType(), Subtarget, DAG, DL); in combineX86ShufflesConstants()
39049 const X86Subtarget &Subtarget) { in combineX86ShufflesRecursively() argument
39266 return getZeroVector(RootVT, Subtarget, DAG, SDLoc(Root)); in combineX86ShufflesRecursively()
39303 Subtarget)) in combineX86ShufflesRecursively()
39310 Ops, Mask, Root, HasVariableMask, DAG, Subtarget)) in combineX86ShufflesRecursively()
39328 Ops, Mask, RootSizeInBits, SDLoc(Root), DAG, Subtarget)) in combineX86ShufflesRecursively()
39381 Op = widenSubVector(Op, false, Subtarget, DAG, SDLoc(Op), in combineX86ShufflesRecursively()
39411 AllowVariablePerLaneMask, DAG, Subtarget); in combineX86ShufflesRecursively()
39418 AllowVariablePerLaneMask, DAG, Subtarget); in combineX86ShufflesRecursively()
39423 const X86Subtarget &Subtarget) { in combineX86ShufflesRecursively() argument
39428 Subtarget); in combineX86ShufflesRecursively()
39621 const X86Subtarget &Subtarget = DAG.getSubtarget<X86Subtarget>(); in combineCommutableSHUFP() local
39622 if (!X86::mayFoldLoad(peekThroughOneUseBitcasts(N0), Subtarget) || in combineCommutableSHUFP()
39623 X86::mayFoldLoad(peekThroughOneUseBitcasts(N1), Subtarget)) in combineCommutableSHUFP()
39849 const X86Subtarget &Subtarget) { in combineTargetShuffle() argument
39896 /*AllowPerLaneVarMask*/ true, DAG, Subtarget)) in combineTargetShuffle()
39968 assert(Subtarget.hasAVX2() && "Expected AVX2"); in combineTargetShuffle()
40166 getZeroVector(VT, Subtarget, DAG, DL), Movl, in combineTargetShuffle()
40554 static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget, in isAddSubOrSubAdd() argument
40560 if (!Subtarget.hasSSE3() || !TLI.isTypeLegal(VT) || in isAddSubOrSubAdd()
40615 const X86Subtarget &Subtarget, in combineShuffleToFMAddSub() argument
40625 if (!Subtarget.hasAnyFMA() || !TLI.isTypeLegal(VT)) in combineShuffleToFMAddSub()
40658 const X86Subtarget &Subtarget, in combineShuffleToAddSubOrFMAddSub() argument
40660 if (SDValue V = combineShuffleToFMAddSub(N, Subtarget, DAG)) in combineShuffleToAddSubOrFMAddSub()
40665 if (!isAddSubOrSubAdd(N, Subtarget, DAG, Opnd0, Opnd1, IsSubAdd)) in combineShuffleToAddSubOrFMAddSub()
40673 if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, 2)) { in combineShuffleToAddSubOrFMAddSub()
40700 const X86Subtarget &Subtarget) { in combineShuffleOfConcatUndef() argument
40701 if (!Subtarget.hasAVX2() || !isa<ShuffleVectorSDNode>(N)) in combineShuffleOfConcatUndef()
40776 const X86Subtarget &Subtarget) { in combineShuffle() argument
40787 if (SDValue AddSub = combineShuffleToAddSubOrFMAddSub(N, Subtarget, DAG)) in combineShuffle()
40792 VT, SDValue(N, 0), dl, DAG, Subtarget, /*IsAfterLegalize*/ true)) in combineShuffle()
40801 if (SDValue ShufConcat = combineShuffleOfConcatUndef(N, DAG, Subtarget)) in combineShuffle()
40806 if (SDValue Shuffle = combineTargetShuffle(Op, DAG, DCI, Subtarget)) in combineShuffle()
40814 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) in combineShuffle()
41016 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op))); in SimplifyDemandedVectorEltsForTargetNode()
41042 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op))); in SimplifyDemandedVectorEltsForTargetNode()
41323 Src = widenSubVector(VT.getSimpleVT(), Src, false, Subtarget, TLO.DAG, in SimplifyDemandedVectorEltsForTargetNode()
41467 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, DL)); in SimplifyDemandedVectorEltsForTargetNode()
41560 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op))); in SimplifyDemandedVectorEltsForTargetNode()
41607 Subtarget); in SimplifyDemandedVectorEltsForTargetNode()
41648 bool Is32BitAVX512 = !Subtarget.is64Bit() && Subtarget.hasAVX512(); in SimplifyDemandedBitsForTargetNode()
41922 if ((BitWidth == 64) && SrcVT.isScalarInteger() && !Subtarget.hasAVX512() && in SimplifyDemandedBitsForTargetNode()
42133 return getZeroVector(VT.getSimpleVT(), Subtarget, DAG, SDLoc(Op)); in SimplifyMultipleUseDemandedBitsForTargetNode()
42272 const X86Subtarget &Subtarget) { in combineBitcastvxi1() argument
42279 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2()) { in combineBitcastvxi1()
42310 if (!Subtarget.hasSSE2() || (Subtarget.hasAVX512() && !PreferMovMsk)) in combineBitcastvxi1()
42335 if (Subtarget.hasAVX() && in combineBitcastvxi1()
42336 checkBitcastSrcVectorSize(Src, 256, Subtarget.hasAVX2())) { in combineBitcastvxi1()
42348 if (Subtarget.hasAVX() && (checkBitcastSrcVectorSize(Src, 256, true) || in combineBitcastvxi1()
42367 if (Subtarget.hasAVX512()) { in combineBitcastvxi1()
42368 if (Subtarget.hasBWI()) in combineBitcastvxi1()
42385 V = getPMOVMSKB(DL, V, DAG, Subtarget); in combineBitcastvxi1()
42419 const X86Subtarget &Subtarget) { in combineCastedMaskArithmetic() argument
42426 if (!Subtarget.hasAVX512()) in combineCastedMaskArithmetic()
42474 const X86Subtarget &Subtarget) { in createMMXBuildVector() argument
42484 if (Subtarget.hasSSE1() && !isa<ConstantFPSDNode>(V)) { in createMMXBuildVector()
42508 if (Subtarget.hasSSE1()) { in createMMXBuildVector()
42555 const X86Subtarget &Subtarget) { in combineBitcastToBoolVector() argument
42574 combineBitcastToBoolVector(NewSrcVT, Src, DL, DAG, Subtarget)) in combineBitcastToBoolVector()
42587 combineBitcastToBoolVector(NewSrcVT, Src, DL, DAG, Subtarget)) in combineBitcastToBoolVector()
42598 if (SDValue N0 = combineBitcastToBoolVector(VT, Src0, DL, DAG, Subtarget)) in combineBitcastToBoolVector()
42599 if (SDValue N1 = combineBitcastToBoolVector(VT, Src1, DL, DAG, Subtarget)) in combineBitcastToBoolVector()
42606 if ((VT == MVT::v8i1 && !Subtarget.hasDQI()) || in combineBitcastToBoolVector()
42607 ((VT == MVT::v32i1 || VT == MVT::v64i1) && !Subtarget.hasBWI())) in combineBitcastToBoolVector()
42611 if (SDValue N0 = combineBitcastToBoolVector(VT, Src0, DL, DAG, Subtarget)) in combineBitcastToBoolVector()
42623 const X86Subtarget &Subtarget) { in combineBitcast() argument
42637 if (SDValue V = combineBitcastvxi1(DAG, VT, N0, dl, Subtarget)) in combineBitcast()
42643 Subtarget.hasAVX512()) { in combineBitcast()
42653 Subtarget.hasAVX512()) { in combineBitcast()
42687 combineBitcastToBoolVector(VT, N0, SDLoc(N), DAG, Subtarget)) in combineBitcast()
42697 if (VT == MVT::i8 && SrcVT == MVT::v8i1 && Subtarget.hasAVX512() && in combineBitcast()
42698 !Subtarget.hasDQI() && N0.getOpcode() == ISD::EXTRACT_SUBVECTOR && in combineBitcast()
42774 return createMMXBuildVector(cast<BuildVectorSDNode>(N0), DAG, Subtarget); in combineBitcast()
42798 if (Subtarget.hasAVX512() && VT.isScalarInteger() && in combineBitcast()
42804 if (Subtarget.hasAVX512() && SrcVT.isScalarInteger() && in combineBitcast()
42817 if (Subtarget.hasAVX512() && SrcVT.isScalarInteger() && in combineBitcast()
42835 (Subtarget.hasBWI() || MovmskVT.getVectorElementType() != MVT::i8)) { in combineBitcast()
42857 if (SDValue V = combineCastedMaskArithmetic(N, DAG, DCI, Subtarget)) in combineBitcast()
42874 if (!((Subtarget.hasSSE1() && VT == MVT::f32) || in combineBitcast()
42875 (Subtarget.hasSSE2() && VT == MVT::f64) || in combineBitcast()
42876 (Subtarget.hasFP16() && VT == MVT::f16) || in combineBitcast()
42877 (Subtarget.hasSSE2() && VT.isInteger() && VT.isVector() && in combineBitcast()
42961 const X86Subtarget &Subtarget) { in createVPDPBUSD() argument
42976 if (Subtarget.hasVNNI() && !Subtarget.hasVLX()) in createVPDPBUSD()
42999 return SplitOpsAndApply(DAG, Subtarget, DL, DpVT, {Zero, DpOp0, DpOp1}, in createVPDPBUSD()
43007 const X86Subtarget &Subtarget) { in createPSADBW() argument
43029 return SplitOpsAndApply(DAG, Subtarget, DL, SadVT, { SadOp0, SadOp1 }, in createPSADBW()
43036 const X86Subtarget &Subtarget) { in combineMinMaxReduction() argument
43038 if (!Subtarget.hasSSE41()) in combineMinMaxReduction()
43110 const X86Subtarget &Subtarget) { in combinePredicateReduction() argument
43112 if (!Subtarget.hasSSE2()) in combinePredicateReduction()
43138 unsigned MaxElts = Subtarget.hasInt256() ? 32 : 16; in combinePredicateReduction()
43173 Movmsk = combineBitcastvxi1(DAG, MovmskVT, Match, DL, Subtarget); in combinePredicateReduction()
43182 (MatchSizeInBits == 256 && Subtarget.hasAVX()))) in combinePredicateReduction()
43196 if (MatchSizeInBits == 256 && BitWidth < 32 && !Subtarget.hasInt256()) { in combinePredicateReduction()
43212 Movmsk = getPMOVMSKB(DL, BitcastLogicOp, DAG, Subtarget); in combinePredicateReduction()
43249 const X86Subtarget &Subtarget) { in combineVPDPBUSDPattern() argument
43250 if (!Subtarget.hasVNNI() && !Subtarget.hasAVXVNNI()) in combineVPDPBUSDPattern()
43289 SDValue DP = createVPDPBUSD(DAG, LHS, RHS, StageBias, DL, Subtarget); in combineVPDPBUSDPattern()
43320 const X86Subtarget &Subtarget) { in combineBasicSADPattern() argument
43322 if (!Subtarget.hasSSE2()) in combineBasicSADPattern()
43364 SDValue SAD = createPSADBW(DAG, Zext0, Zext1, DL, Subtarget); in combineBasicSADPattern()
43397 const X86Subtarget &Subtarget) { in combineExtractWithShuffle() argument
43491 auto GetLegalExtract = [&Subtarget, &DAG, &dl](SDValue Vec, EVT VecVT, in combineExtractWithShuffle()
43506 ((Idx == 0 && Subtarget.hasSSE2()) || Subtarget.hasSSE41())) { in combineExtractWithShuffle()
43511 if ((VecVT == MVT::v8i16 && Subtarget.hasSSE2()) || in combineExtractWithShuffle()
43512 (VecVT == MVT::v16i8 && Subtarget.hasSSE41())) { in combineExtractWithShuffle()
43595 const X86Subtarget &Subtarget) { in scalarizeExtEltFP() argument
43623 if (!(VT == MVT::f16 && Subtarget.hasFP16()) && VT != MVT::f32 && in scalarizeExtEltFP()
43695 const X86Subtarget &Subtarget) { in combineArithReduction() argument
43699 if (!Subtarget.hasSSE2()) in combineArithReduction()
43724 if (ZeroExtend && Subtarget.hasSSE41()) { in combineArithReduction()
43801 getZeroVector(MVT::v16i8, Subtarget, DAG, DL)); in combineArithReduction()
43813 Subtarget.hasAVX512())) { in combineArithReduction()
43827 Rdx = SplitOpsAndApply(DAG, Subtarget, DL, SadVT, {Rdx}, PSADBWBuilder); in combineArithReduction()
43849 if (!shouldUseHorizontalOp(true, DAG, Subtarget)) in combineArithReduction()
43858 if (((VecVT == MVT::v16i16 || VecVT == MVT::v8i32) && Subtarget.hasSSSE3()) || in combineArithReduction()
43859 ((VecVT == MVT::v8f32 || VecVT == MVT::v4f64) && Subtarget.hasSSE3())) { in combineArithReduction()
43866 if (!((VecVT == MVT::v8i16 || VecVT == MVT::v4i32) && Subtarget.hasSSSE3()) && in combineArithReduction()
43867 !((VecVT == MVT::v4f32 || VecVT == MVT::v2f64) && Subtarget.hasSSE3())) in combineArithReduction()
43884 const X86Subtarget &Subtarget) { in combineExtractVectorElt() argument
43885 if (SDValue NewOp = combineExtractWithShuffle(N, DAG, DCI, Subtarget)) in combineExtractVectorElt()
43962 if (SDValue SAD = combineBasicSADPattern(N, DAG, Subtarget)) in combineExtractVectorElt()
43965 if (SDValue VPDPBUSD = combineVPDPBUSDPattern(N, DAG, Subtarget)) in combineExtractVectorElt()
43969 if (SDValue Cmp = combinePredicateReduction(N, DAG, Subtarget)) in combineExtractVectorElt()
43973 if (SDValue MinMax = combineMinMaxReduction(N, DAG, Subtarget)) in combineExtractVectorElt()
43977 if (SDValue V = combineArithReduction(N, DAG, Subtarget)) in combineExtractVectorElt()
43980 if (SDValue V = scalarizeExtEltFP(N, DAG, Subtarget)) in combineExtractVectorElt()
44012 combineBitcastvxi1(DAG, BCVT, InputVector, dl, Subtarget)) { in combineExtractVectorElt()
44064 TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) { in combineToExtendBoolVectorInReg() argument
44070 if (!Subtarget.hasSSE2() || Subtarget.hasAVX512()) in combineToExtendBoolVectorInReg()
44111 } else if (Subtarget.hasAVX2() && NumElts < EltSizeInBits && in combineToExtendBoolVectorInReg()
44164 const X86Subtarget &Subtarget) { in combineVSelectWithAllOnesOrZeros() argument
44269 const X86Subtarget &Subtarget) { in narrowVectorSelect() argument
44293 return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, { Cond, TVal, FVal }, in narrowVectorSelect()
44376 const X86Subtarget &Subtarget) { in combineVSelectToBLENDV() argument
44405 if (VT.is128BitVector() && !Subtarget.hasSSE41()) in combineVSelectToBLENDV()
44408 if (VT == MVT::v32i8 && !Subtarget.hasAVX2()) in combineVSelectToBLENDV()
44483 SelectionDAG &DAG, const X86Subtarget &Subtarget) { in combineLogicBlendIntoConditionalNegate() argument
44530 const X86Subtarget &Subtarget) { in combineSelect() argument
44554 DL, DAG, Subtarget)) in combineSelect()
44604 VT != MVT::f80 && VT != MVT::f128 && !isSoftFP16(VT, Subtarget) && in combineSelect()
44606 (Subtarget.hasSSE2() || in combineSelect()
44607 (Subtarget.hasSSE1() && VT.getScalarType() == MVT::f32))) { in combineSelect()
44756 if (Subtarget.hasAVX512() && N->getOpcode() == ISD::SELECT && in combineSelect()
44777 if (Subtarget.hasAVX512() && !Subtarget.hasBWI() && CondVT.isVector() && in combineSelect()
44789 if (Subtarget.hasAVX512() && CondVT.isVector() && in combineSelect()
44898 Subtarget.hasAVX512() && CondVT.getVectorElementType() == MVT::i1 && in combineSelect()
44914 ISD::SIGN_EXTEND, DL, ExtCondVT, Cond, DAG, DCI, Subtarget)) { in combineSelect()
44921 if (!TLI.isTypeLegal(VT) || isSoftFP16(VT, Subtarget)) in combineSelect()
44924 if (SDValue V = combineVSelectWithAllOnesOrZeros(N, DAG, DCI, Subtarget)) in combineSelect()
44927 if (SDValue V = combineVSelectToBLENDV(N, DAG, DCI, Subtarget)) in combineSelect()
44930 if (SDValue V = narrowVectorSelect(N, DAG, Subtarget)) in combineSelect()
44954 (DCI.isBeforeLegalize() || (VT != MVT::v64i1 || Subtarget.is64Bit()))) { in combineSelect()
44983 if (DCI.isBeforeLegalize() && !Subtarget.hasAVX512() && in combineSelect()
45005 TLI.isTypeLegal(VT) && ((Subtarget.hasAVX() && EltBitWidth == 32) || in combineSelect()
45006 (Subtarget.hasAVX2() && EltBitWidth == 64) || in combineSelect()
45007 (Subtarget.hasXOP())); in combineSelect()
45040 const X86Subtarget &Subtarget) { in combineSetCCAtomicArith() argument
45127 auto LockOp = lowerAtomicArithWithLOCK(AtomicSub, DAG, Subtarget); in combineSetCCAtomicArith()
45149 SDValue LockOp = lowerAtomicArithWithLOCK(CmpLHS, DAG, Subtarget); in combineSetCCAtomicArith()
45390 const X86Subtarget &Subtarget) { in combinePTESTCC() argument
45489 Res = getPMOVMSKB(DL, Res, DAG, Subtarget); in combinePTESTCC()
45493 Res = getPMOVMSKB(DL, Res, DAG, Subtarget); in combinePTESTCC()
45511 if (CC == X86::COND_E && OpVT.is128BitVector() && Subtarget.hasAVX()) { in combinePTESTCC()
45533 const X86Subtarget &Subtarget) { in combineSetCCMOVMSK() argument
45628 if (IsAllOf && Subtarget.hasSSE41() && IsOneUse) { in combineSetCCMOVMSK()
45682 if (CmpBits >= 16 && Subtarget.hasInt256() && in combineSetCCMOVMSK()
45742 const X86Subtarget &Subtarget) { in combineSetCCEFLAGS() argument
45750 if (SDValue R = combinePTESTCC(EFLAGS, CC, DAG, Subtarget)) in combineSetCCEFLAGS()
45753 if (SDValue R = combineSetCCMOVMSK(EFLAGS, CC, DAG, Subtarget)) in combineSetCCEFLAGS()
45756 return combineSetCCAtomicArith(EFLAGS, CC, DAG, Subtarget); in combineSetCCEFLAGS()
45762 const X86Subtarget &Subtarget) { in combineCMov() argument
45776 if (SDValue Flags = combineSetCCEFLAGS(Cond, CC, DAG, Subtarget)) { in combineCMov()
45778 (FalseOp.getValueType() == MVT::f64 && !Subtarget.hasSSE2()) || in combineCMov()
45779 (FalseOp.getValueType() == MVT::f32 && !Subtarget.hasSSE1())) || in combineCMov()
45780 !Subtarget.canUseCMOV() || hasFPCMov(CC)) { in combineCMov()
46049 const X86Subtarget &Subtarget) { in reduceVMULWidth() argument
46052 if (!Subtarget.hasSSE2()) in reduceVMULWidth()
46060 if (Subtarget.hasSSE41() && (OptForMinSize || !Subtarget.isPMULLDSlow())) in reduceVMULWidth()
46208 const X86Subtarget &Subtarget) { in combineMulToPMADDWD() argument
46209 if (!Subtarget.hasSSE2()) in combineMulToPMADDWD()
46212 if (Subtarget.isPMADDWDSlow()) in combineMulToPMADDWD()
46228 if (32 <= (2 * NumElts) && Subtarget.hasAVX512() && !Subtarget.hasBWI()) in combineMulToPMADDWD()
46236 if (!Subtarget.hasSSE41() && in combineMulToPMADDWD()
46249 if (!Subtarget.hasSSE41() && in combineMulToPMADDWD()
46278 if (Src.getScalarValueSizeInBits() < 16 && !Subtarget.hasSSE41()) { in combineMulToPMADDWD()
46315 return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, {N0, N1}, in combineMulToPMADDWD()
46320 const X86Subtarget &Subtarget) { in combineMulToPMULDQ() argument
46321 if (!Subtarget.hasSSE2()) in combineMulToPMULDQ()
46337 if (Subtarget.hasSSE41() && DAG.ComputeNumSignBits(N0) > 32 && in combineMulToPMULDQ()
46343 return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, { N0, N1 }, in combineMulToPMULDQ()
46354 return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, { N0, N1 }, in combineMulToPMULDQ()
46363 const X86Subtarget &Subtarget) { in combineMul() argument
46366 if (SDValue V = combineMulToPMADDWD(N, DAG, Subtarget)) in combineMul()
46369 if (SDValue V = combineMulToPMULDQ(N, DAG, Subtarget)) in combineMul()
46373 return reduceVMULWidth(N, DAG, Subtarget); in combineMul()
46457 } else if (!Subtarget.slowLEA()) in combineMul()
46514 const X86Subtarget &Subtarget) { in combineShiftToPMULH() argument
46519 if (!Subtarget.hasSSE2()) in combineShiftToPMULH()
46622 const X86Subtarget &Subtarget) { in combineShiftRightArithmetic() argument
46628 if (SDValue V = combineShiftToPMULH(N, DAG, Subtarget)) in combineShiftRightArithmetic()
46679 const X86Subtarget &Subtarget) { in combineShiftRightLogical() argument
46684 if (SDValue V = combineShiftToPMULH(N, DAG, Subtarget)) in combineShiftRightLogical()
46731 const X86Subtarget &Subtarget) { in combineHorizOpWithShuffle() argument
46835 if (VT.is256BitVector() && Subtarget.hasInt256()) { in combineHorizOpWithShuffle()
46876 const X86Subtarget &Subtarget) { in combineVectorPack() argument
46948 if (SDValue V = combineHorizOpWithShuffle(N, DAG, Subtarget)) in combineVectorPack()
46953 if (Subtarget.hasAVX512() && in combineVectorPack()
46959 if (Subtarget.hasVLX()) in combineVectorPack()
47002 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) in combineVectorPack()
47010 const X86Subtarget &Subtarget) { in combineVectorHADDSUB() argument
47015 if (!shouldUseHorizontalOp(true, DAG, Subtarget)) { in combineVectorHADDSUB()
47050 if (SDValue V = combineHorizOpWithShuffle(N, DAG, Subtarget)) in combineVectorHADDSUB()
47058 const X86Subtarget &Subtarget) { in combineVectorShiftVar() argument
47089 const X86Subtarget &Subtarget) { in combineVectorShiftImm() argument
47150 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) in combineVectorShiftImm()
47190 const X86Subtarget &Subtarget) { in combineVectorInsert() argument
47208 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) in combineVectorInsert()
47220 const X86Subtarget &Subtarget) { in combineCompareEqual() argument
47225 if (Subtarget.hasSSE2() && isAndOrOfSetCCs(SDValue(N, 0U), opcode)) { in combineCompareEqual()
47241 (VT == MVT::f16 && Subtarget.hasFP16())) { in combineCompareEqual()
47278 if (Subtarget.hasAVX512()) { in combineCompareEqual()
47297 if (is64BitFP && !Subtarget.is64Bit()) { in combineCompareEqual()
47433 const X86Subtarget &Subtarget) { in PromoteMaskArithmetic() argument
47477 const X86Subtarget &Subtarget) { in convertIntLogicToFPLogic() argument
47493 if (N00Type != N10Type || !((Subtarget.hasSSE1() && N00Type == MVT::f32) || in convertIntLogicToFPLogic()
47494 (Subtarget.hasSSE2() && N00Type == MVT::f64) || in convertIntLogicToFPLogic()
47495 (Subtarget.hasFP16() && N00Type == MVT::f16))) in convertIntLogicToFPLogic()
47513 if (!Subtarget.hasAVX() && in convertIntLogicToFPLogic()
47618 const X86Subtarget &Subtarget) { in combineAndMaskToShift() argument
47635 supportedVectorShiftWithImm(VT.getSimpleVT(), Subtarget, ISD::SRA)) { in combineAndMaskToShift()
47664 if (!supportedVectorShiftWithImm(VT.getSimpleVT(), Subtarget, ISD::SRL)) in combineAndMaskToShift()
47698 static bool hasBZHI(const X86Subtarget &Subtarget, MVT VT) { in hasBZHI() argument
47699 if (Subtarget.hasBMI2() && VT.isScalarInteger()) { in hasBZHI()
47702 case 64: return Subtarget.is64Bit() ? true : false; in hasBZHI()
47720 const X86Subtarget &Subtarget) { in combineAndLoadToBZHI() argument
47725 if (!hasBZHI(Subtarget, VT)) in combineAndLoadToBZHI()
47802 const X86Subtarget &Subtarget) { in combineScalarAndWithMaskSetcc() argument
47843 !(Subtarget.hasVLX() || SetccVT.is512BitVector())) in combineScalarAndWithMaskSetcc()
47846 if (!(Subtarget.hasBWI() || SetccVT.getScalarSizeInBits() >= 32)) in combineScalarAndWithMaskSetcc()
47862 const X86Subtarget &Subtarget) { in combineAnd() argument
47870 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) { in combineAnd()
47878 if (VT == MVT::i64 && Subtarget.is64Bit() && !isa<ConstantSDNode>(N1)) { in combineAnd()
47898 SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget); in combineAnd()
47911 if (SDValue V = combineScalarAndWithMaskSetcc(N, DAG, Subtarget)) in combineAnd()
47920 if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, DCI, Subtarget)) in combineAnd()
47926 if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget)) in combineAnd()
47932 if (SDValue ShiftRight = combineAndMaskToShift(N, DAG, Subtarget)) in combineAnd()
47935 if (SDValue R = combineAndLoadToBZHI(N, DAG, Subtarget)) in combineAnd()
47979 if (!(Subtarget.hasBMI2() && !ContainsNOT && VT.getSizeInBits() >= 32)) in combineAnd()
47988 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) in combineAnd()
48072 /*AllowVarPerLaneMask*/ true, DAG, Subtarget)) in combineAnd()
48083 const X86Subtarget &Subtarget) { in canonicalizeBitSelect() argument
48098 if (!(Subtarget.hasXOP() || useVPTERNLOG(Subtarget, VT) || in canonicalizeBitSelect()
48122 if (useVPTERNLOG(Subtarget, VT)) { in canonicalizeBitSelect()
48133 DAG, Subtarget); in canonicalizeBitSelect()
48185 const X86Subtarget &Subtarget) { in combineLogicBlendIntoPBLENDV() argument
48189 if (!((VT.is128BitVector() && Subtarget.hasSSE2()) || in combineLogicBlendIntoPBLENDV()
48190 (VT.is256BitVector() && Subtarget.hasInt256()))) in combineLogicBlendIntoPBLENDV()
48213 DAG, Subtarget)) in combineLogicBlendIntoPBLENDV()
48217 if (!Subtarget.hasSSE41()) in combineLogicBlendIntoPBLENDV()
48221 if (Subtarget.hasVLX()) in combineLogicBlendIntoPBLENDV()
48262 const X86Subtarget &Subtarget) { in combineOrCmpEqZeroToCtlzSrl() argument
48263 if (DCI.isBeforeLegalize() || !Subtarget.getTargetLowering()->isCtlzFast()) in combineOrCmpEqZeroToCtlzSrl()
48390 const X86Subtarget &Subtarget) { in combineOr() argument
48398 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) { in combineOr()
48414 SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget); in combineOr()
48434 if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, DCI, Subtarget)) in combineOr()
48440 if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget)) in combineOr()
48443 if (SDValue R = canonicalizeBitSelect(N, DAG, Subtarget)) in combineOr()
48446 if (SDValue R = combineLogicBlendIntoPBLENDV(N, DAG, Subtarget)) in combineOr()
48478 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) in combineOr()
48506 if (!Subtarget.hasBMI() && VT.isScalarInteger() && VT != MVT::i1) in combineOr()
48573 const X86Subtarget &Subtarget) { in foldVectorXorShiftIntoCmp() argument
48583 case MVT::v2i64: if (!Subtarget.hasSSE2()) return SDValue(); break; in foldVectorXorShiftIntoCmp()
48587 case MVT::v4i64: if (!Subtarget.hasAVX2()) return SDValue(); break; in foldVectorXorShiftIntoCmp()
48709 const X86Subtarget &Subtarget) { in combineTruncateWithSat() argument
48710 if (!Subtarget.hasSSE2() || !VT.isVector()) in combineTruncateWithSat()
48721 if (Subtarget.hasBWI() && !Subtarget.useAVX512Regs() && in combineTruncateWithSat()
48726 DL, DAG, Subtarget); in combineTruncateWithSat()
48738 bool PreferAVX512 = ((Subtarget.hasAVX512() && InSVT == MVT::i32) || in combineTruncateWithSat()
48739 (Subtarget.hasBWI() && InSVT == MVT::i16)) && in combineTruncateWithSat()
48741 (Subtarget.hasVLX() || InVT.getSizeInBits() > 256) && in combineTruncateWithSat()
48742 !(!Subtarget.useAVX512Regs() && VT.getSizeInBits() >= 256); in combineTruncateWithSat()
48755 DAG, Subtarget); in combineTruncateWithSat()
48758 Subtarget); in combineTruncateWithSat()
48761 } else if (SVT == MVT::i8 || Subtarget.hasSSE41()) in combineTruncateWithSat()
48763 Subtarget); in combineTruncateWithSat()
48767 Subtarget); in combineTruncateWithSat()
48772 Subtarget.hasAVX512() && (InSVT != MVT::i16 || Subtarget.hasBWI()) && in combineTruncateWithSat()
48787 if (!Subtarget.hasVLX() && !InVT.is512BitVector()) { in combineTruncateWithSat()
48813 const X86Subtarget &Subtarget, in detectAVGPattern() argument
48830 if (!Subtarget.hasSSE2()) in detectAVGPattern()
48895 SDValue Res = SplitOpsAndApply(DAG, Subtarget, DL, Pow2VT, Ops, AVGBuilder); in detectAVGPattern()
48962 const X86Subtarget &Subtarget) { in combineLoad() argument
48976 ((Ld->isNonTemporal() && !Subtarget.hasInt256() && in combineLoad()
49008 if (Ext == ISD::NON_EXTLOAD && !Subtarget.hasAVX512() && RegVT.isVector() && in combineLoad()
49024 if (Ext == ISD::NON_EXTLOAD && Subtarget.hasAVX() && Ld->isSimple() && in combineLoad()
49132 const X86Subtarget &Subtarget) { in reduceMaskedLoadToScalarLoad() argument
49151 if (EltVT == MVT::i64 && !Subtarget.is64Bit()) { in reduceMaskedLoadToScalarLoad()
49221 const X86Subtarget &Subtarget) { in combineMaskedLoad() argument
49230 reduceMaskedLoadToScalarLoad(Mld, DAG, DCI, Subtarget)) in combineMaskedLoad()
49234 if (!Subtarget.hasAVX512()) in combineMaskedLoad()
49268 const X86Subtarget &Subtarget) { in reduceMaskedStoreToScalarStore() argument
49284 if (EltVT == MVT::i64 && !Subtarget.is64Bit()) { in reduceMaskedStoreToScalarStore()
49300 const X86Subtarget &Subtarget) { in combineMaskedStore() argument
49312 if (SDValue ScalarStore = reduceMaskedStoreToScalarStore(Mst, DAG, Subtarget)) in combineMaskedStore()
49348 const X86Subtarget &Subtarget) { in combineStore() argument
49357 if (!Subtarget.hasAVX512() && VT == StVT && VT.isVector() && in combineStore()
49370 if (VT == MVT::v1i1 && VT == StVT && Subtarget.hasAVX512() && in combineStore()
49384 Subtarget.hasAVX512()) { in combineStore()
49400 if (!DCI.isBeforeLegalize() && VT == MVT::v64i1 && !Subtarget.is64Bit()) { in combineStore()
49457 if (VT.is128BitVector() && Subtarget.hasSSE2()) { in combineStore()
49458 MVT NTVT = Subtarget.hasSSE4A() in combineStore()
49467 if (!St->isTruncatingStore() && VT == MVT::v16i8 && !Subtarget.hasBWI() && in combineStore()
49529 Subtarget, dl)) in combineStore()
49575 !Subtarget.useSoftFloat() && !NoImplicitFloatOps && Subtarget.hasSSE2(); in combineStore()
49576 if ((VT == MVT::i64 && F64IsLegal && !Subtarget.is64Bit()) && in combineStore()
49607 if (VT == MVT::i64 && F64IsLegal && !Subtarget.is64Bit() && in combineStore()
49626 const X86Subtarget &Subtarget) { in combineVEXTRACT_STORE() argument
49661 SelectionDAG &DAG, const X86Subtarget &Subtarget, in isHorizontalBinOp() argument
49820 if (!IsIdentityPostShuffle && !Subtarget.hasAVX2() && VT.isFloatingPoint() && in isHorizontalBinOp()
49839 DAG, Subtarget)) in isHorizontalBinOp()
49849 const X86Subtarget &Subtarget) { in combineToHorizontalAddSub() argument
49858 if ((Subtarget.hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) || in combineToHorizontalAddSub()
49859 (Subtarget.hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) { in combineToHorizontalAddSub()
49863 if (isHorizontalBinOp(HorizOpcode, LHS, RHS, DAG, Subtarget, IsAdd, in combineToHorizontalAddSub()
49875 if (Subtarget.hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32 || in combineToHorizontalAddSub()
49880 if (isHorizontalBinOp(HorizOpcode, LHS, RHS, DAG, Subtarget, IsAdd, in combineToHorizontalAddSub()
49886 SDValue HorizBinOp = SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, in combineToHorizontalAddSub()
49914 const X86Subtarget &Subtarget) { in combineFMulcFCMulc() argument
49974 const X86Subtarget &Subtarget) { in combineFaddCFmul() argument
50000 if (N->getOpcode() != ISD::FADD || !Subtarget.hasFP16() || in combineFaddCFmul()
50058 const X86Subtarget &Subtarget) { in combineFaddFsub() argument
50059 if (SDValue HOp = combineToHorizontalAddSub(N, DAG, Subtarget)) in combineFaddFsub()
50062 if (SDValue COp = combineFaddCFmul(N, DAG, Subtarget)) in combineFaddFsub()
50074 const X86Subtarget &Subtarget, in combineTruncatedArithmetic() argument
50152 const X86Subtarget &Subtarget, in combineVectorTruncationWithPACKUS() argument
50161 return truncateVectorWithPACK(X86ISD::PACKUS, OutVT, In, DL, DAG, Subtarget); in combineVectorTruncationWithPACKUS()
50166 const X86Subtarget &Subtarget, in combineVectorTruncationWithPACKSS() argument
50173 return truncateVectorWithPACK(X86ISD::PACKSS, OutVT, In, DL, DAG, Subtarget); in combineVectorTruncationWithPACKSS()
50182 const X86Subtarget &Subtarget) { in combineVectorTruncation() argument
50195 if (!Subtarget.hasSSE2() || Subtarget.hasAVX512()) in combineVectorTruncation()
50206 if (Subtarget.hasSSSE3() && NumElems == 8) { in combineVectorTruncation()
50210 (OutSVT == MVT::i8 || !Subtarget.hasSSE41() || Subtarget.hasInt256())) in combineVectorTruncation()
50218 if (Subtarget.hasSSE41() || OutSVT == MVT::i8) in combineVectorTruncation()
50219 return combineVectorTruncationWithPACKUS(N, DL, Subtarget, DAG); in combineVectorTruncation()
50221 return combineVectorTruncationWithPACKSS(N, DL, Subtarget, DAG); in combineVectorTruncation()
50231 const X86Subtarget &Subtarget) { in combineVectorSignBitsTruncation() argument
50233 if (!Subtarget.hasSSE2()) in combineVectorSignBitsTruncation()
50263 if (Subtarget.hasAVX512() && in combineVectorSignBitsTruncation()
50264 !(!Subtarget.useAVX512Regs() && VT.is256BitVector() && in combineVectorSignBitsTruncation()
50275 unsigned NumPackedZeroBits = Subtarget.hasSSE41() ? NumPackedSignBits : 8; in combineVectorSignBitsTruncation()
50282 return truncateVectorWithPACK(X86ISD::PACKUS, VT, In, DL, DAG, Subtarget); in combineVectorSignBitsTruncation()
50296 return truncateVectorWithPACK(X86ISD::PACKSS, VT, In, DL, DAG, Subtarget); in combineVectorSignBitsTruncation()
50307 Subtarget); in combineVectorSignBitsTruncation()
50322 SelectionDAG &DAG, const X86Subtarget &Subtarget) { in combinePMULH() argument
50328 if (!Subtarget.hasSSE2()) in combinePMULH()
50379 if (IsUnsigned && !IsTruncateFree && Subtarget.hasInt256() && in combinePMULH()
50380 !(Subtarget.hasAVX512() && !Subtarget.hasBWI() && VT.is256BitVector()) && in combinePMULH()
50406 const X86Subtarget &Subtarget, in detectPMADDUBSW() argument
50408 if (!VT.isVector() || !Subtarget.hasSSSE3()) in detectPMADDUBSW()
50532 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { ZExtIn, SExtIn }, in detectPMADDUBSW()
50537 const X86Subtarget &Subtarget) { in combineTruncate() argument
50543 if (SDValue V = combineTruncatedArithmetic(N, DAG, Subtarget, DL)) in combineTruncate()
50547 if (SDValue Avg = detectAVGPattern(Src, VT, DAG, Subtarget, DL)) in combineTruncate()
50551 if (SDValue PMAdd = detectPMADDUBSW(Src, VT, DAG, Subtarget, DL)) in combineTruncate()
50555 if (SDValue Val = combineTruncateWithSat(Src, VT, DL, DAG, Subtarget)) in combineTruncate()
50559 if (SDValue V = combinePMULH(Src, VT, DL, DAG, Subtarget)) in combineTruncate()
50571 if (SDValue V = combineVectorSignBitsTruncation(N, DL, DAG, Subtarget)) in combineTruncate()
50574 return combineVectorTruncation(N, DAG, Subtarget); in combineTruncate()
50747 const X86Subtarget &Subtarget) { in combineFneg() argument
50766 Arg->getFlags().hasNoSignedZeros() && Subtarget.hasAnyFMA()) { in combineFneg()
50806 if (!Op.hasOneUse() || !Subtarget.hasAnyFMA() || !isTypeLegal(VT) || in getNegatedExpression()
50850 const X86Subtarget &Subtarget) { in lowerX86FPLogicOp() argument
50853 if (!VT.isVector() || !Subtarget.hasSSE2()) in lowerX86FPLogicOp()
50894 const X86Subtarget &Subtarget) { in combineXor() argument
50900 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) { in combineXor()
50907 if (SDValue Cmp = foldVectorXorShiftIntoCmp(N, DAG, Subtarget)) in combineXor()
50916 if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, DCI, Subtarget)) in combineXor()
50967 return combineFneg(N, DAG, DCI, Subtarget); in combineXor()
50972 const X86Subtarget &Subtarget) { in combineBEXTR() argument
50998 const X86Subtarget &Subtarget) { in getNullFPConstForNullVal() argument
51003 return getZeroVector(V.getSimpleValueType(), Subtarget, DAG, SDLoc(V)); in getNullFPConstForNullVal()
51009 const X86Subtarget &Subtarget) { in combineFAndFNotToFAndn() argument
51016 if (!((VT == MVT::f32 && Subtarget.hasSSE1()) || in combineFAndFNotToFAndn()
51017 (VT == MVT::f64 && Subtarget.hasSSE2()) || in combineFAndFNotToFAndn()
51018 (VT == MVT::v4f32 && Subtarget.hasSSE1() && !Subtarget.hasSSE2()))) in combineFAndFNotToFAndn()
51041 const X86Subtarget &Subtarget) { in combineFAnd() argument
51043 if (SDValue V = getNullFPConstForNullVal(N->getOperand(0), DAG, Subtarget)) in combineFAnd()
51047 if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget)) in combineFAnd()
51050 if (SDValue V = combineFAndFNotToFAndn(N, DAG, Subtarget)) in combineFAnd()
51053 return lowerX86FPLogicOp(N, DAG, Subtarget); in combineFAnd()
51058 const X86Subtarget &Subtarget) { in combineFAndn() argument
51064 if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget)) in combineFAndn()
51067 return lowerX86FPLogicOp(N, DAG, Subtarget); in combineFAndn()
51073 const X86Subtarget &Subtarget) { in combineFOr() argument
51084 if (SDValue NewVal = combineFneg(N, DAG, DCI, Subtarget)) in combineFOr()
51087 return lowerX86FPLogicOp(N, DAG, Subtarget); in combineFOr()
51113 const X86Subtarget &Subtarget) { in combineFMinNumFMaxNum() argument
51114 if (Subtarget.useSoftFloat()) in combineFMinNumFMaxNum()
51120 if (!((Subtarget.hasSSE1() && VT == MVT::f32) || in combineFMinNumFMaxNum()
51121 (Subtarget.hasSSE2() && VT == MVT::f64) || in combineFMinNumFMaxNum()
51122 (Subtarget.hasFP16() && VT == MVT::f16) || in combineFMinNumFMaxNum()
51250 const X86Subtarget &Subtarget) { in combineAndnp() argument
51293 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) in combineAndnp()
51458 const X86Subtarget &Subtarget) { in combineSignExtendInReg() argument
51481 if (N00.getOpcode() == ISD::LOAD && Subtarget.hasInt256()) in combineSignExtendInReg()
51487 if (SDValue Promote = PromoteMaskArithmetic(N0.getNode(), DAG, Subtarget)) in combineSignExtendInReg()
51505 const X86Subtarget &Subtarget) { in promoteExtBeforeAdd() argument
51624 const X86Subtarget &Subtarget) { in combineExtSetcc() argument
51630 if (!Subtarget.hasAVX512() || !VT.isVector() || N0.getOpcode() != ISD::SETCC) in combineExtSetcc()
51644 if (Size > 256 && Subtarget.useAVX512Regs()) in combineExtSetcc()
51669 const X86Subtarget &Subtarget) { in combineSext() argument
51697 if (SDValue V = combineExtSetcc(N, DAG, Subtarget)) in combineSext()
51701 DAG, DCI, Subtarget)) in combineSext()
51705 if (SDValue R = PromoteMaskArithmetic(N, DAG, Subtarget)) in combineSext()
51712 if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget)) in combineSext()
51720 const X86Subtarget &Subtarget) { in combineFMA() argument
51745 !Subtarget.hasAnyFMA()) && in combineFMA()
51746 !(ScalarVT == MVT::f16 && Subtarget.hasFP16())) in combineFMA()
51825 const X86Subtarget &Subtarget) { in combineZext() argument
51852 if (SDValue V = combineExtSetcc(N, DAG, Subtarget)) in combineZext()
51856 DAG, DCI, Subtarget)) in combineZext()
51860 if (SDValue R = PromoteMaskArithmetic(N, DAG, Subtarget)) in combineZext()
51863 if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget)) in combineZext()
51866 if (SDValue R = combineOrCmpEqZeroToCtlzSrl(N, DAG, DCI, Subtarget)) in combineZext()
51927 const X86Subtarget &Subtarget) { in combineVectorSizedSetCCEquality() argument
51967 if (!Subtarget.useSoftFloat() && !NoImplicitFloatOps && in combineVectorSizedSetCCEquality()
51968 ((OpSize == 128 && Subtarget.hasSSE2()) || in combineVectorSizedSetCCEquality()
51969 (OpSize == 256 && Subtarget.hasAVX()) || in combineVectorSizedSetCCEquality()
51970 (OpSize == 512 && Subtarget.useAVX512Regs()))) { in combineVectorSizedSetCCEquality()
51971 bool HasPT = Subtarget.hasSSE41(); in combineVectorSizedSetCCEquality()
51976 bool PreferKOT = Subtarget.preferMaskRegisters(); in combineVectorSizedSetCCEquality()
51977 bool NeedZExt = PreferKOT && !Subtarget.hasVLX() && OpSize != 512; in combineVectorSizedSetCCEquality()
51988 if (Subtarget.hasBWI()) { in combineVectorSizedSetCCEquality()
52076 const X86Subtarget &Subtarget) { in combineSetCC() argument
52085 if (SDValue V = combineVectorSizedSetCCEquality(N, DAG, Subtarget)) in combineSetCC()
52091 MatchVectorAllZeroTest(LHS, CC, DL, Subtarget, DAG, X86CC)) in combineSetCC()
52191 if (Subtarget.hasAVX512() && !Subtarget.hasBWI() && VT.isVector() && in combineSetCC()
52201 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32 && in combineSetCC()
52203 return LowerVSETCC(SDValue(N, 0), Subtarget, DAG); in combineSetCC()
52220 const X86Subtarget &Subtarget) { in combineMOVMSK() argument
52243 if (Subtarget.hasSSE2() && Src.getOpcode() == ISD::BITCAST && in combineMOVMSK()
52304 const X86Subtarget &Subtarget) { in combineX86GatherScatter() argument
52332 MemOp->getChain(), Subtarget); in combineX86GatherScatter()
52336 NewIndex, NewScale, MemOp->getChain(), Subtarget); in combineX86GatherScatter()
52490 const X86Subtarget &Subtarget) { in combineX86SetCC() argument
52496 if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget)) in combineX86SetCC()
52504 const X86Subtarget &Subtarget) { in combineBrCond() argument
52512 if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget)) { in combineBrCond()
52613 const X86Subtarget &Subtarget) { in combineUIntToFP() argument
52670 const X86Subtarget &Subtarget) { in combineSIntToFP() argument
52719 if (InVT.getScalarSizeInBits() > 32 && !Subtarget.hasDQI()) { in combineSIntToFP()
52749 if (!Subtarget.useSoftFloat() && Subtarget.hasX87() && in combineSIntToFP()
52759 if (Subtarget.hasDQI() && VT != MVT::f80) in combineSIntToFP()
52763 Op0.hasOneUse() && !Subtarget.is64Bit() && InVT == MVT::i64) { in combineSIntToFP()
52765 Subtarget.getTargetLowering()->BuildFILD( in combineSIntToFP()
53282 const X86Subtarget &Subtarget) { in matchPMADDWD() argument
53294 if (!Subtarget.hasSSE2()) in matchPMADDWD()
53382 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { N0, N1 }, PMADDBuilder); in matchPMADDWD()
53390 const X86Subtarget &Subtarget) { in matchPMADDWD_2() argument
53391 if (!Subtarget.hasSSE2()) in matchPMADDWD_2()
53520 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { In0, In1 }, in matchPMADDWD_2()
53573 const X86Subtarget &Subtarget) { in pushAddIntoCmovOfConsts() argument
53600 if (X86::mayFoldLoad(OtherOp, Subtarget)) in pushAddIntoCmovOfConsts()
53639 const X86Subtarget &Subtarget) { in combineAdd() argument
53645 if (SDValue Select = pushAddIntoCmovOfConsts(N, DAG, Subtarget)) in combineAdd()
53648 if (SDValue MAdd = matchPMADDWD(DAG, Op0, Op1, DL, VT, Subtarget)) in combineAdd()
53650 if (SDValue MAdd = matchPMADDWD_2(DAG, Op0, Op1, DL, VT, Subtarget)) in combineAdd()
53656 if (SDValue V = combineToHorizontalAddSub(N, DAG, Subtarget)) in combineAdd()
53735 const X86Subtarget &Subtarget) { in combineSub() argument
53768 if (SDValue V = combineToHorizontalAddSub(N, DAG, Subtarget)) in combineSub()
53794 const X86Subtarget &Subtarget) { in combineVectorCompare() argument
53814 const X86Subtarget &Subtarget) { in combineConcatVectorOps() argument
53815 assert(Subtarget.hasAVX() && "AVX assumed for concat_vectors"); in combineConcatVectorOps()
53824 return getZeroVector(VT, Subtarget, DAG, DL); in combineConcatVectorOps()
53831 (VT.is256BitVector() || (VT.is512BitVector() && Subtarget.hasAVX512()))) { in combineConcatVectorOps()
53857 (Subtarget.hasAVX2() || in combineConcatVectorOps()
53859 VT.getScalarType(), Subtarget))) in combineConcatVectorOps()
53867 (Subtarget.hasAVX2() || in combineConcatVectorOps()
53869 X86::mayFoldLoad(Op0.getOperand(0), Subtarget))) && in combineConcatVectorOps()
53964 Subtarget.hasInt256() && Op0.getOperand(1) == Ops[1].getOperand(1)) { in combineConcatVectorOps()
53987 if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) || in combineConcatVectorOps()
53988 (VT.is512BitVector() && Subtarget.useBWIRegs()))) { in combineConcatVectorOps()
54029 if (VT == MVT::v4i64 && !Subtarget.hasInt256() && in combineConcatVectorOps()
54034 SDValue Zero = getZeroVector(MVT::v8i32, Subtarget, DAG, DL); in combineConcatVectorOps()
54049 if (((VT.is256BitVector() && Subtarget.hasInt256()) || in combineConcatVectorOps()
54050 (VT.is512BitVector() && Subtarget.useAVX512Regs() && in combineConcatVectorOps()
54051 (EltSizeInBits >= 32 || Subtarget.useBWIRegs()))) && in combineConcatVectorOps()
54062 if (VT.is512BitVector() && Subtarget.useAVX512Regs() && in combineConcatVectorOps()
54091 (VT.isFloatingPoint() || Subtarget.hasInt256())) { in combineConcatVectorOps()
54102 ((VT.is256BitVector() && Subtarget.hasInt256()) || in combineConcatVectorOps()
54103 (VT.is512BitVector() && Subtarget.useBWIRegs())) && in combineConcatVectorOps()
54115 (VT.getScalarSizeInBits() >= 32 || Subtarget.hasInt256()) && in combineConcatVectorOps()
54132 const X86TargetLowering *TLI = Subtarget.getTargetLowering(); in combineConcatVectorOps()
54137 EltsFromConsecutiveLoads(VT, Ops, DL, DAG, Subtarget, false)) in combineConcatVectorOps()
54164 const X86Subtarget &Subtarget) { in combineCONCAT_VECTORS() argument
54173 if (Subtarget.hasAVX() && TLI.isTypeLegal(VT) && TLI.isTypeLegal(SrcVT)) { in combineCONCAT_VECTORS()
54176 DCI, Subtarget)) in combineCONCAT_VECTORS()
54185 const X86Subtarget &Subtarget) { in combineINSERT_SUBVECTOR() argument
54206 return getZeroVector(OpVT, Subtarget, DAG, dl); in combineINSERT_SUBVECTOR()
54215 getZeroVector(OpVT, Subtarget, DAG, dl), in combineINSERT_SUBVECTOR()
54233 getZeroVector(OpVT, Subtarget, DAG, dl), in combineINSERT_SUBVECTOR()
54268 combineConcatVectorOps(dl, OpVT, SubVectorOps, DAG, DCI, Subtarget)) in combineINSERT_SUBVECTOR()
54279 getZeroVector(OpVT, Subtarget, DAG, dl), in combineINSERT_SUBVECTOR()
54378 const X86Subtarget &Subtarget) { in combineEXTRACT_SUBVECTOR() argument
54402 if (Subtarget.hasAVX() && !Subtarget.hasAVX2() && in combineEXTRACT_SUBVECTOR()
54428 return getZeroVector(VT, Subtarget, DAG, SDLoc(N)); in combineEXTRACT_SUBVECTOR()
54483 return getZeroVector(VT, Subtarget, DAG, SDLoc(N)); in combineEXTRACT_SUBVECTOR()
54505 if (InOpcode == ISD::UINT_TO_FP && Subtarget.hasVLX() && in combineEXTRACT_SUBVECTOR()
54541 if (IdxVal == 0 && InOpcode == ISD::TRUNCATE && Subtarget.hasVLX() && in combineEXTRACT_SUBVECTOR()
54645 const X86Subtarget &Subtarget) { in combinePMULDQ() argument
54718 const X86Subtarget &Subtarget) { in combineEXTEND_VECTOR_INREG() argument
54776 Subtarget.hasSSE41()) { in combineEXTEND_VECTOR_INREG()
54779 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) in combineEXTEND_VECTOR_INREG()
54805 const X86Subtarget &Subtarget) { in combineFP16_TO_FP() argument
54806 if (Subtarget.useSoftFloat() || !Subtarget.hasF16C()) in combineFP16_TO_FP()
54827 const X86Subtarget &Subtarget) { in combineFP_EXTEND() argument
54828 if (!Subtarget.hasF16C() || Subtarget.useSoftFloat()) in combineFP_EXTEND()
54831 if (Subtarget.hasFP16()) in combineFP_EXTEND()
54938 const X86Subtarget &Subtarget) { in combineFP_ROUND() argument
54939 if (!Subtarget.hasF16C() || Subtarget.useSoftFloat()) in combineFP_ROUND()
54942 if (Subtarget.hasFP16()) in combineFP_ROUND()
55034 return combineExtractVectorElt(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55036 return combineCONCAT_VECTORS(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55038 return combineINSERT_SUBVECTOR(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55040 return combineEXTRACT_SUBVECTOR(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55043 case X86ISD::BLENDV: return combineSelect(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55044 case ISD::BITCAST: return combineBitcast(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55045 case X86ISD::CMOV: return combineCMov(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55047 case ISD::ADD: return combineAdd(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55048 case ISD::SUB: return combineSub(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55053 case ISD::MUL: return combineMul(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55055 case ISD::SRA: return combineShiftRightArithmetic(N, DAG, Subtarget); in PerformDAGCombine()
55056 case ISD::SRL: return combineShiftRightLogical(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55057 case ISD::AND: return combineAnd(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55058 case ISD::OR: return combineOr(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55059 case ISD::XOR: return combineXor(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55061 case X86ISD::BEXTRI: return combineBEXTR(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55062 case ISD::LOAD: return combineLoad(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55063 case ISD::MLOAD: return combineMaskedLoad(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55064 case ISD::STORE: return combineStore(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55065 case ISD::MSTORE: return combineMaskedStore(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55067 return combineVEXTRACT_STORE(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55070 return combineSIntToFP(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55073 return combineUIntToFP(N, DAG, Subtarget); in PerformDAGCombine()
55075 case ISD::FSUB: return combineFaddFsub(N, DAG, Subtarget); in PerformDAGCombine()
55077 case X86ISD::VFMULC: return combineFMulcFCMulc(N, DAG, Subtarget); in PerformDAGCombine()
55078 case ISD::FNEG: return combineFneg(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55079 case ISD::TRUNCATE: return combineTruncate(N, DAG, Subtarget); in PerformDAGCombine()
55081 case X86ISD::ANDNP: return combineAndnp(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55082 case X86ISD::FAND: return combineFAnd(N, DAG, Subtarget); in PerformDAGCombine()
55083 case X86ISD::FANDN: return combineFAndn(N, DAG, Subtarget); in PerformDAGCombine()
55085 case X86ISD::FOR: return combineFOr(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55089 case ISD::FMAXNUM: return combineFMinNumFMaxNum(N, DAG, Subtarget); in PerformDAGCombine()
55103 case ISD::ZERO_EXTEND: return combineZext(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55104 case ISD::SIGN_EXTEND: return combineSext(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55105 case ISD::SIGN_EXTEND_INREG: return combineSignExtendInReg(N, DAG, Subtarget); in PerformDAGCombine()
55109 return combineEXTEND_VECTOR_INREG(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55110 case ISD::SETCC: return combineSetCC(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55111 case X86ISD::SETCC: return combineX86SetCC(N, DAG, Subtarget); in PerformDAGCombine()
55112 case X86ISD::BRCOND: return combineBrCond(N, DAG, Subtarget); in PerformDAGCombine()
55114 case X86ISD::PACKUS: return combineVectorPack(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55118 case X86ISD::FHSUB: return combineVectorHADDSUB(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55122 return combineVectorShiftVar(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55126 return combineVectorShiftImm(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55129 case X86ISD::PINSRW: return combineVectorInsert(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55164 case ISD::VECTOR_SHUFFLE: return combineShuffle(N, DAG, DCI,Subtarget); in PerformDAGCombine()
55176 case ISD::STRICT_FMA: return combineFMA(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55181 case X86ISD::MOVMSK: return combineMOVMSK(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55184 return combineX86GatherScatter(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55188 case X86ISD::PCMPGT: return combineVectorCompare(N, DAG, Subtarget); in PerformDAGCombine()
55190 case X86ISD::PMULUDQ: return combinePMULDQ(N, DAG, DCI, Subtarget); in PerformDAGCombine()
55195 case ISD::FP16_TO_FP: return combineFP16_TO_FP(N, DAG, Subtarget); in PerformDAGCombine()
55197 case ISD::FP_EXTEND: return combineFP_EXTEND(N, DAG, Subtarget); in PerformDAGCombine()
55199 case ISD::FP_ROUND: return combineFP_ROUND(N, DAG, Subtarget); in PerformDAGCombine()
55319 if (X86::mayFoldLoad(N0, Subtarget) && IsFoldableRMW(N0, Op)) in IsDesirableToPromoteOp()
55334 if (X86::mayFoldLoad(N1, Subtarget) && in IsDesirableToPromoteOp()
55338 if (X86::mayFoldLoad(N0, Subtarget) && in IsDesirableToPromoteOp()
55603 if (type->isX86_MMXTy() && Subtarget.hasMMX()) in getSingleConstraintMatchWeight()
55614 if (((type->getPrimitiveSizeInBits() == 128) && Subtarget.hasSSE1()) || in getSingleConstraintMatchWeight()
55615 ((type->getPrimitiveSizeInBits() == 256) && Subtarget.hasAVX()) || in getSingleConstraintMatchWeight()
55616 ((type->getPrimitiveSizeInBits() == 512) && Subtarget.hasAVX512())) in getSingleConstraintMatchWeight()
55621 if ((type->getPrimitiveSizeInBits() == 64) && Subtarget.hasAVX512()) in getSingleConstraintMatchWeight()
55626 if (type->isX86_MMXTy() && Subtarget.hasMMX()) in getSingleConstraintMatchWeight()
55633 if (!Subtarget.hasSSE2()) in getSingleConstraintMatchWeight()
55639 if ((type->getPrimitiveSizeInBits() == 512) && Subtarget.hasAVX512()) in getSingleConstraintMatchWeight()
55643 if (((type->getPrimitiveSizeInBits() == 128) && Subtarget.hasSSE1()) || in getSingleConstraintMatchWeight()
55644 ((type->getPrimitiveSizeInBits() == 256) && Subtarget.hasAVX())) in getSingleConstraintMatchWeight()
55649 if ((type->getPrimitiveSizeInBits() == 64) && Subtarget.hasAVX512()) in getSingleConstraintMatchWeight()
55719 if (Subtarget.hasSSE1()) in LowerXConstraint()
55796 (Subtarget.is64Bit() && C->getZExtValue() == 0xffffffff)) { in LowerAsmOperandForConstraint()
55874 if ((Subtarget.isPICStyleGOT() || Subtarget.isPICStyleStubPIC()) && in LowerAsmOperandForConstraint()
55884 Subtarget.classifyGlobalReference(GA->getGlobal()))) in LowerAsmOperandForConstraint()
55942 if (Subtarget.is64Bit()) in getRegForInlineAsmConstraint()
55944 assert((Subtarget.is32Bit() || Subtarget.is16Bit()) && in getRegForInlineAsmConstraint()
55952 if (Subtarget.hasAVX512()) { in getRegForInlineAsmConstraint()
55960 if (Subtarget.hasBWI()) { in getRegForInlineAsmConstraint()
55968 if (Subtarget.is64Bit()) { in getRegForInlineAsmConstraint()
55987 (!VT.isVector() && !Subtarget.is64Bit())) in getRegForInlineAsmConstraint()
55999 (!VT.isVector() && !Subtarget.is64Bit())) in getRegForInlineAsmConstraint()
56010 (!VT.isVector() && !Subtarget.is64Bit())) in getRegForInlineAsmConstraint()
56026 if (!Subtarget.hasMMX()) break; in getRegForInlineAsmConstraint()
56030 if (!Subtarget.hasSSE1()) break; in getRegForInlineAsmConstraint()
56037 if (VConstraint && Subtarget.hasFP16()) in getRegForInlineAsmConstraint()
56042 if (VConstraint && Subtarget.hasVLX()) in getRegForInlineAsmConstraint()
56047 if (VConstraint && Subtarget.hasVLX()) in getRegForInlineAsmConstraint()
56051 if (Subtarget.is64Bit()) { in getRegForInlineAsmConstraint()
56052 if (VConstraint && Subtarget.hasVLX()) in getRegForInlineAsmConstraint()
56059 if (!Subtarget.hasFP16()) in getRegForInlineAsmConstraint()
56069 if (VConstraint && Subtarget.hasVLX()) in getRegForInlineAsmConstraint()
56074 if (!Subtarget.hasFP16()) in getRegForInlineAsmConstraint()
56083 if (VConstraint && Subtarget.hasVLX()) in getRegForInlineAsmConstraint()
56085 if (Subtarget.hasAVX()) in getRegForInlineAsmConstraint()
56089 if (!Subtarget.hasFP16()) in getRegForInlineAsmConstraint()
56098 if (!Subtarget.hasAVX512()) break; in getRegForInlineAsmConstraint()
56114 if (!Subtarget.hasMMX()) break; in getRegForInlineAsmConstraint()
56117 if (!Subtarget.hasSSE1()) break; in getRegForInlineAsmConstraint()
56122 if (!Subtarget.hasFP16()) in getRegForInlineAsmConstraint()
56132 if (!Subtarget.hasFP16()) in getRegForInlineAsmConstraint()
56145 if (!Subtarget.hasFP16()) in getRegForInlineAsmConstraint()
56154 if (Subtarget.hasAVX()) in getRegForInlineAsmConstraint()
56158 if (!Subtarget.hasFP16()) in getRegForInlineAsmConstraint()
56167 if (Subtarget.hasAVX512()) in getRegForInlineAsmConstraint()
56174 if (Subtarget.hasAVX512()) { in getRegForInlineAsmConstraint()
56182 if (Subtarget.hasBWI()) { in getRegForInlineAsmConstraint()
56242 if (!Subtarget.is64Bit() && in getRegForInlineAsmConstraint()
56250 if (!Subtarget.hasAVX512() && isFRClass(*Res.second) && in getRegForInlineAsmConstraint()
56276 bool is64Bit = Subtarget.is64Bit(); in getRegForInlineAsmConstraint()
56398 if (!Subtarget.is64Bit()) in initializeSplitCSR()
56410 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo(); in insertCopiesSplitCSR()
56415 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); in insertCopiesSplitCSR()
56447 return Subtarget.is64Bit(); in supportSwiftError()
56459 if (Subtarget.isOSWindows() || in hasInlineStackProbe()
56485 if (!Subtarget.isOSWindows() || Subtarget.isTargetMachO() || in getStackProbeSymbolName()
56491 if (Subtarget.is64Bit()) in getStackProbeSymbolName()
56492 return Subtarget.isTargetCygMing() ? "___chkstk_ms" : "__chkstk"; in getStackProbeSymbolName()
56493 return Subtarget.isTargetCygMing() ? "_alloca" : "_chkstk"; in getStackProbeSymbolName()