Lines Matching refs:Subtarget
84 : TargetLowering(TM), Subtarget(STI) { in RISCVTargetLowering()
86 RISCVABI::ABI ABI = Subtarget.getTargetABI(); in RISCVTargetLowering()
90 !Subtarget.hasStdExtF()) { in RISCVTargetLowering()
94 ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32; in RISCVTargetLowering()
96 !Subtarget.hasStdExtD()) { in RISCVTargetLowering()
100 ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32; in RISCVTargetLowering()
117 MVT XLenVT = Subtarget.getXLenVT(); in RISCVTargetLowering()
121 if (Subtarget.is64Bit() && RV64LegalI32) in RISCVTargetLowering()
124 if (Subtarget.hasStdExtZfhmin()) in RISCVTargetLowering()
126 if (Subtarget.hasStdExtZfbfmin()) in RISCVTargetLowering()
128 if (Subtarget.hasStdExtF()) in RISCVTargetLowering()
130 if (Subtarget.hasStdExtD()) in RISCVTargetLowering()
132 if (Subtarget.hasStdExtZhinxmin()) in RISCVTargetLowering()
134 if (Subtarget.hasStdExtZfinx()) in RISCVTargetLowering()
136 if (Subtarget.hasStdExtZdinx()) { in RISCVTargetLowering()
137 if (Subtarget.is64Bit()) in RISCVTargetLowering()
163 if (Subtarget.hasVInstructions()) { in RISCVTargetLowering()
167 unsigned MinElts = RISCV::RVVBitsPerBlock / Subtarget.getELen(); in RISCVTargetLowering()
191 !Subtarget.hasVInstructionsI64()) in RISCVTargetLowering()
196 if (Subtarget.hasVInstructionsF16Minimal()) in RISCVTargetLowering()
200 if (Subtarget.hasVInstructionsBF16()) in RISCVTargetLowering()
204 if (Subtarget.hasVInstructionsF32()) in RISCVTargetLowering()
208 if (Subtarget.hasVInstructionsF64()) in RISCVTargetLowering()
212 if (Subtarget.useRVVForFixedLengthVectors()) { in RISCVTargetLowering()
216 const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo(); in RISCVTargetLowering()
245 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
249 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
259 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
271 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb()) in RISCVTargetLowering()
274 if (Subtarget.is64Bit()) { in RISCVTargetLowering()
292 if (!Subtarget.hasStdExtM() && !Subtarget.hasStdExtZmmul()) { in RISCVTargetLowering()
294 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
296 } else if (Subtarget.is64Bit()) { in RISCVTargetLowering()
304 if (!Subtarget.hasStdExtM()) { in RISCVTargetLowering()
307 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
310 } else if (Subtarget.is64Bit()) { in RISCVTargetLowering()
316 if (RV64LegalI32 && Subtarget.is64Bit()) { in RISCVTargetLowering()
330 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) { in RISCVTargetLowering()
331 if (!RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
333 } else if (Subtarget.hasVendorXTHeadBb()) { in RISCVTargetLowering()
334 if (Subtarget.is64Bit()) in RISCVTargetLowering()
337 } else if (Subtarget.hasVendorXCVbitmanip()) { in RISCVTargetLowering()
341 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
348 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() || in RISCVTargetLowering()
349 Subtarget.hasVendorXTHeadBb()) in RISCVTargetLowering()
352 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
354 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() || in RISCVTargetLowering()
355 Subtarget.hasVendorXTHeadBb()) in RISCVTargetLowering()
360 if (Subtarget.hasVendorXCVbitmanip()) { in RISCVTargetLowering()
365 Subtarget.hasStdExtZbkb() ? Custom : Expand); in RISCVTargetLowering()
368 if (Subtarget.hasStdExtZbb()) { in RISCVTargetLowering()
371 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
375 if (Subtarget.is64Bit()) { in RISCVTargetLowering()
381 } else if (!Subtarget.hasVendorXCVbitmanip()) { in RISCVTargetLowering()
383 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
387 if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() || in RISCVTargetLowering()
388 Subtarget.hasVendorXCVbitmanip()) { in RISCVTargetLowering()
391 if (Subtarget.is64Bit()) { in RISCVTargetLowering()
394 Subtarget.hasStdExtZbb() ? Legal : Promote); in RISCVTargetLowering()
395 if (!Subtarget.hasStdExtZbb()) in RISCVTargetLowering()
402 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
406 if (!RV64LegalI32 && Subtarget.is64Bit() && in RISCVTargetLowering()
407 !Subtarget.hasShortForwardBranchOpt()) in RISCVTargetLowering()
411 if (Subtarget.hasShortForwardBranchOpt()) in RISCVTargetLowering()
414 if (!Subtarget.hasVendorXTHeadCondMov()) in RISCVTargetLowering()
417 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
441 if (Subtarget.hasStdExtZfhminOrZhinxmin()) in RISCVTargetLowering()
455 if (Subtarget.hasStdExtZfbfmin()) { in RISCVTargetLowering()
471 if (Subtarget.hasStdExtZfhminOrZhinxmin()) { in RISCVTargetLowering()
472 if (Subtarget.hasStdExtZfhOrZhinx()) { in RISCVTargetLowering()
475 Subtarget.hasStdExtZfa() ? Legal : Custom); in RISCVTargetLowering()
495 Subtarget.hasStdExtZfa() ? Legal : Promote); in RISCVTargetLowering()
511 if (Subtarget.is64Bit()) in RISCVTargetLowering()
514 if (!Subtarget.hasStdExtZfa()) in RISCVTargetLowering()
518 if (Subtarget.hasStdExtFOrZfinx()) { in RISCVTargetLowering()
521 Subtarget.hasStdExtZfa() ? Legal : Custom); in RISCVTargetLowering()
534 Subtarget.isSoftFPABI() ? LibCall : Custom); in RISCVTargetLowering()
538 if (Subtarget.hasStdExtZfa()) in RISCVTargetLowering()
544 if (Subtarget.hasStdExtFOrZfinx() && Subtarget.is64Bit()) in RISCVTargetLowering()
547 if (Subtarget.hasStdExtDOrZdinx()) { in RISCVTargetLowering()
550 if (Subtarget.hasStdExtZfa()) { in RISCVTargetLowering()
556 if (Subtarget.is64Bit()) in RISCVTargetLowering()
578 Subtarget.isSoftFPABI() ? LibCall : Custom); in RISCVTargetLowering()
583 if (Subtarget.is64Bit()) { in RISCVTargetLowering()
590 if (Subtarget.hasStdExtFOrZfinx()) { in RISCVTargetLowering()
598 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
613 if (Subtarget.is64Bit()) in RISCVTargetLowering()
619 Subtarget.is64Bit() ? Legal : Custom); in RISCVTargetLowering()
623 if (Subtarget.is64Bit()) in RISCVTargetLowering()
626 if (Subtarget.hasStdExtZicbop()) { in RISCVTargetLowering()
630 if (Subtarget.hasStdExtA()) { in RISCVTargetLowering()
631 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen()); in RISCVTargetLowering()
633 } else if (Subtarget.hasForcedAtomics()) { in RISCVTargetLowering()
634 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen()); in RISCVTargetLowering()
643 if (Subtarget.hasVInstructions()) { in RISCVTargetLowering()
647 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
655 if (Subtarget.is64Bit()) in RISCVTargetLowering()
702 if (!Subtarget.is64Bit()) { in RISCVTargetLowering()
795 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV()) in RISCVTargetLowering()
866 if (Subtarget.hasStdExtZvkb()) { in RISCVTargetLowering()
874 if (Subtarget.hasStdExtZvbb()) { in RISCVTargetLowering()
1019 if (Subtarget.hasVInstructionsF16()) { in RISCVTargetLowering()
1025 } else if (Subtarget.hasVInstructionsF16Minimal()) { in RISCVTargetLowering()
1059 if (Subtarget.hasVInstructionsF32()) { in RISCVTargetLowering()
1068 if (Subtarget.hasVInstructionsF64()) { in RISCVTargetLowering()
1078 if (Subtarget.useRVVForFixedLengthVectors()) { in RISCVTargetLowering()
1163 if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) { in RISCVTargetLowering()
1186 if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV()) in RISCVTargetLowering()
1208 if (Subtarget.hasStdExtZvkb()) in RISCVTargetLowering()
1211 if (Subtarget.hasStdExtZvbb()) { in RISCVTargetLowering()
1245 !Subtarget.hasVInstructionsF16()) { in RISCVTargetLowering()
1329 if (Subtarget.hasStdExtZfhminOrZhinxmin()) in RISCVTargetLowering()
1331 if (Subtarget.hasStdExtFOrZfinx()) in RISCVTargetLowering()
1333 if (Subtarget.hasStdExtDOrZdinx()) in RISCVTargetLowering()
1338 if (Subtarget.hasStdExtA()) { in RISCVTargetLowering()
1340 if (RV64LegalI32 && Subtarget.is64Bit()) in RISCVTargetLowering()
1344 if (Subtarget.hasForcedAtomics()) { in RISCVTargetLowering()
1354 if (Subtarget.hasVendorXTHeadMemIdx()) { in RISCVTargetLowering()
1363 if (Subtarget.is64Bit()) { in RISCVTargetLowering()
1371 const Align FunctionAlignment(Subtarget.hasStdExtCOrZca() ? 2 : 4); in RISCVTargetLowering()
1374 setPrefFunctionAlignment(Subtarget.getPrefFunctionAlignment()); in RISCVTargetLowering()
1375 setPrefLoopAlignment(Subtarget.getPrefLoopAlignment()); in RISCVTargetLowering()
1380 if (Subtarget.is64Bit()) in RISCVTargetLowering()
1383 if (Subtarget.hasStdExtFOrZfinx()) in RISCVTargetLowering()
1386 if (Subtarget.hasStdExtZbb()) in RISCVTargetLowering()
1389 if (Subtarget.hasStdExtZbs() && Subtarget.is64Bit()) in RISCVTargetLowering()
1392 if (Subtarget.hasStdExtZbkb()) in RISCVTargetLowering()
1394 if (Subtarget.hasStdExtZfhminOrZhinxmin()) in RISCVTargetLowering()
1396 if (Subtarget.hasStdExtFOrZfinx()) in RISCVTargetLowering()
1399 if (Subtarget.hasVInstructions()) in RISCVTargetLowering()
1406 if (Subtarget.hasVendorXTHeadMemPair()) in RISCVTargetLowering()
1408 if (Subtarget.useRVVForFixedLengthVectors()) in RISCVTargetLowering()
1423 if (Subtarget.hasVInstructions() && in getSetCCResultType()
1424 (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors())) in getSetCCResultType()
1430 return Subtarget.getXLenVT(); in getVPExplicitVectorLengthTy()
1437 if (!Subtarget.hasVInstructions()) in shouldExpandGetVectorLength()
1443 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.getXLenVT()) in shouldExpandGetVectorLength()
1447 if (VF < RISCV::RVVBitsPerBlock / Subtarget.getELen()) in shouldExpandGetVectorLength()
1451 if (Subtarget.getRealMinVLen() < RISCV::RVVBitsPerBlock) in shouldExpandGetVectorLength()
1733 if (Subtarget.hasVInstructions() && isa<VectorType>(Ty)) in isLegalAddressingMode()
1768 if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy()) in isTruncateFree()
1802 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64; in isSExtCheaperThanZExt()
1806 return Subtarget.is64Bit() && CI->getType()->isIntegerTy(32); in signExtendConstant()
1810 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXCVbitmanip(); in isCheapToSpeculateCttz()
1814 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() || in isCheapToSpeculateCtlz()
1815 Subtarget.hasVendorXCVbitmanip(); in isCheapToSpeculateCtlz()
1826 if (!Subtarget.hasStdExtZbs() && !Subtarget.hasVendorXTHeadBs()) in isMaskAndCmp0FoldingBeneficial()
1841 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) && in hasAndNotCompare()
1847 if (Subtarget.hasStdExtZbs()) in hasBitTest()
1851 if (Subtarget.hasVendorXTHeadBs()) in hasBitTest()
1860 if (!VT.isVector() || !Subtarget.hasVInstructions()) in shouldFoldSelectWithIdentityConstant()
1874 if (BitSize > Subtarget.getXLen()) in shouldConvertConstantLoadToIntImm()
1886 if (!Subtarget.hasFastUnalignedAccess() && in shouldConvertConstantLoadToIntImm()
1887 !Subtarget.enableUnalignedScalarMem()) in shouldConvertConstantLoadToIntImm()
1896 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, Subtarget); in shouldConvertConstantLoadToIntImm()
1897 return Seq.size() <= Subtarget.getMaxBuildIntsCost(); in shouldConvertConstantLoadToIntImm()
1954 if (!I->getType()->isVectorTy() || !Subtarget.hasVInstructions()) in canSplatOperand()
2003 if (!I->getType()->isVectorTy() || !Subtarget.hasVInstructions()) in shouldSinkOperands()
2078 if (!Subtarget.hasStdExtZfa()) in getLegalZfaFPImm()
2083 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh(); in getLegalZfaFPImm()
2087 assert(Subtarget.hasStdExtD() && "Expect D extension"); in getLegalZfaFPImm()
2106 IsLegalVT = Subtarget.hasStdExtZfhminOrZhinxmin(); in isFPImmLegal()
2108 IsLegalVT = Subtarget.hasStdExtFOrZfinx(); in isFPImmLegal()
2110 IsLegalVT = Subtarget.hasStdExtDOrZdinx(); in isFPImmLegal()
2112 IsLegalVT = Subtarget.hasStdExtZfbfmin(); in isFPImmLegal()
2121 if (Subtarget.getXLen() < VT.getScalarSizeInBits()) { in isFPImmLegal()
2134 1 + RISCVMatInt::getIntMatCost(Imm.bitcastToAPInt(), Subtarget.getXLen(), in isFPImmLegal()
2135 Subtarget); in isFPImmLegal()
2177 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() && in getRegisterTypeForCallingConv()
2178 !Subtarget.hasStdExtZfhminOrZhinxmin()) in getRegisterTypeForCallingConv()
2183 if (RV64LegalI32 && Subtarget.is64Bit() && PartVT == MVT::i32) in getRegisterTypeForCallingConv()
2194 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() && in getNumRegistersForCallingConv()
2195 !Subtarget.hasStdExtZfhminOrZhinxmin()) in getNumRegistersForCallingConv()
2207 if (RV64LegalI32 && Subtarget.is64Bit() && IntermediateVT == MVT::i32) in getVectorTypeBreakdownForCallingConv()
2210 if (RV64LegalI32 && Subtarget.is64Bit() && RegisterVT == MVT::i32) in getVectorTypeBreakdownForCallingConv()
2394 return !Subtarget.useRVVForFixedLengthVectors() || in mergeStoresAfterLegalization()
2403 return Subtarget.is64Bit() ? Subtarget.hasVInstructionsI64() : true; in isLegalElementTypeForRVV()
2409 return Subtarget.hasVInstructionsI64(); in isLegalElementTypeForRVV()
2411 return Subtarget.hasVInstructionsF16(); in isLegalElementTypeForRVV()
2413 return Subtarget.hasVInstructionsF32(); in isLegalElementTypeForRVV()
2415 return Subtarget.hasVInstructionsF64(); in isLegalElementTypeForRVV()
2440 const RISCVSubtarget &Subtarget) { in useRVVForFixedLengthVectorVT() argument
2442 if (!Subtarget.useRVVForFixedLengthVectors()) in useRVVForFixedLengthVectorVT()
2452 unsigned MinVLen = Subtarget.getRealMinVLen(); in useRVVForFixedLengthVectorVT()
2472 if (!Subtarget.hasVInstructionsI64()) in useRVVForFixedLengthVectorVT()
2476 if (!Subtarget.hasVInstructionsF16Minimal()) in useRVVForFixedLengthVectorVT()
2480 if (!Subtarget.hasVInstructionsF32()) in useRVVForFixedLengthVectorVT()
2484 if (!Subtarget.hasVInstructionsF64()) in useRVVForFixedLengthVectorVT()
2490 if (EltVT.getSizeInBits() > Subtarget.getELen()) in useRVVForFixedLengthVectorVT()
2495 if (LMul > Subtarget.getMaxLMULForFixedLengthVectors()) in useRVVForFixedLengthVectorVT()
2507 return ::useRVVForFixedLengthVectorVT(VT, Subtarget); in useRVVForFixedLengthVectorVT()
2512 const RISCVSubtarget &Subtarget) { in getContainerForFixedLengthVector() argument
2515 useRVVForFixedLengthVectorVT(VT, Subtarget)) && in getContainerForFixedLengthVector()
2518 unsigned MinVLen = Subtarget.getRealMinVLen(); in getContainerForFixedLengthVector()
2519 unsigned MaxELen = Subtarget.getELen(); in getContainerForFixedLengthVector()
2546 const RISCVSubtarget &Subtarget) { in getContainerForFixedLengthVector() argument
2548 Subtarget); in getContainerForFixedLengthVector()
2557 const RISCVSubtarget &Subtarget) { in convertToScalableVector() argument
2563 SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT()); in convertToScalableVector()
2569 const RISCVSubtarget &Subtarget) { in convertFromScalableVector() argument
2575 SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT()); in convertFromScalableVector()
2597 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) { in getVLOp() argument
2602 RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget); in getVLOp()
2604 return DAG.getRegister(RISCV::X0, Subtarget.getXLenVT()); in getVLOp()
2606 return DAG.getConstant(NumElts, DL, Subtarget.getXLenVT()); in getVLOp()
2611 const RISCVSubtarget &Subtarget) { in getDefaultScalableVLOps() argument
2613 SDValue VL = DAG.getRegister(RISCV::X0, Subtarget.getXLenVT()); in getDefaultScalableVLOps()
2620 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) { in getDefaultVLOps() argument
2622 SDValue VL = getVLOp(NumElts, ContainerVT, DL, DAG, Subtarget); in getDefaultVLOps()
2633 const RISCVSubtarget &Subtarget) { in getDefaultVLOps() argument
2636 Subtarget); in getDefaultVLOps()
2638 return getDefaultScalableVLOps(ContainerVT, DL, DAG, Subtarget); in getDefaultVLOps()
2644 return DAG.getElementCount(DL, Subtarget.getXLenVT(), in computeVLMax()
2650 const RISCVSubtarget &Subtarget) { in computeVLMAXBounds() argument
2656 unsigned VectorBitsMax = Subtarget.getRealMaxVLen(); in computeVLMAXBounds()
2660 unsigned VectorBitsMin = Subtarget.getRealMinVLen(); in computeVLMAXBounds()
2685 unsigned DLenFactor = Subtarget.getDLenFactor(); in getLMULCost()
2697 Cost = divideCeil(VT.getSizeInBits(), Subtarget.getRealMinVLen() / DLenFactor); in getLMULCost()
2734 const RISCVSubtarget &Subtarget) { in lowerFP_TO_INT_SAT() argument
2748 if ((Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) || in lowerFP_TO_INT_SAT()
2765 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT())); in lowerFP_TO_INT_SAT()
2794 DstContainerVT = getContainerForFixedLengthVector(DAG, DstVT, Subtarget); in lowerFP_TO_INT_SAT()
2795 SrcContainerVT = getContainerForFixedLengthVector(DAG, SrcVT, Subtarget); in lowerFP_TO_INT_SAT()
2799 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget); in lowerFP_TO_INT_SAT()
2804 auto [Mask, VL] = getDefaultVLOps(DstVT, DstContainerVT, DL, DAG, Subtarget); in lowerFP_TO_INT_SAT()
2824 DAG.getConstant(0, DL, Subtarget.getXLenVT()), VL); in lowerFP_TO_INT_SAT()
2829 Res = convertFromScalableVector(DstVT, Res, DAG, Subtarget); in lowerFP_TO_INT_SAT()
2869 const RISCVSubtarget &Subtarget) { in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND() argument
2879 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2880 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2888 Subtarget); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2891 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2922 MVT XLenVT = Subtarget.getXLenVT(); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2970 return convertFromScalableVector(VT, Truncated, DAG, Subtarget); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2978 const RISCVSubtarget &Subtarget) { in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND() argument
2986 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
2987 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
2990 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3030 MVT XLenVT = Subtarget.getXLenVT(); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3073 Truncated = convertFromScalableVector(VT, Truncated, DAG, Subtarget); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3079 const RISCVSubtarget &Subtarget) { in lowerFTRUNC_FCEIL_FFLOOR_FROUND() argument
3082 return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget); in lowerFTRUNC_FCEIL_FFLOOR_FROUND()
3102 DAG.getTargetConstant(FRM, DL, Subtarget.getXLenVT())); in lowerFTRUNC_FCEIL_FFLOOR_FROUND()
3107 const RISCVSubtarget &Subtarget) { in lowerVectorXRINT() argument
3116 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVectorXRINT()
3117 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget); in lowerVectorXRINT()
3120 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVectorXRINT()
3127 return convertFromScalableVector(VT, Truncated, DAG, Subtarget); in lowerVectorXRINT()
3131 getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, in getVSlidedown() argument
3137 SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT()); in getVSlidedown()
3143 getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, in getVSlideup() argument
3149 SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT()); in getVSlideup()
3305 const RISCVSubtarget &Subtarget) { in matchSplatAsGather() argument
3316 if (Idx.getValueType() != Subtarget.getXLenVT()) in matchSplatAsGather()
3321 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in matchSplatAsGather()
3322 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in matchSplatAsGather()
3325 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in matchSplatAsGather()
3333 return convertFromScalableVector(VT, Gather, DAG, Subtarget); in matchSplatAsGather()
3346 const RISCVSubtarget &Subtarget) { in lowerBuildVectorViaDominantValues() argument
3350 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerBuildVectorViaDominantValues()
3353 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerBuildVectorViaDominantValues()
3355 MVT XLenVT = Subtarget.getXLenVT(); in lowerBuildVectorViaDominantValues()
3413 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerBuildVectorViaDominantValues()
3420 Vec = convertFromScalableVector(VT, Vec, DAG, Subtarget); in lowerBuildVectorViaDominantValues()
3453 const RISCVSubtarget &Subtarget) { in lowerBuildVectorOfConstants() argument
3457 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerBuildVectorOfConstants()
3460 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerBuildVectorOfConstants()
3462 MVT XLenVT = Subtarget.getXLenVT(); in lowerBuildVectorOfConstants()
3468 return convertFromScalableVector(VT, VMClr, DAG, Subtarget); in lowerBuildVectorOfConstants()
3473 return convertFromScalableVector(VT, VMSet, DAG, Subtarget); in lowerBuildVectorOfConstants()
3484 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.getXLen()); in lowerBuildVectorOfConstants()
3485 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.getELen()); in lowerBuildVectorOfConstants()
3549 return convertFromScalableVector(VT, Splat, DAG, Subtarget); in lowerBuildVectorOfConstants()
3584 getContainerForFixedLengthVector(DAG, VIDVT, Subtarget); in lowerBuildVectorOfConstants()
3590 VID = convertFromScalableVector(VIDVT, VID, DAG, Subtarget); in lowerBuildVectorOfConstants()
3626 (Subtarget.getRealMinVLen() >= VT.getSizeInBits() * NumElts) ? NumElts : 1; in lowerBuildVectorOfConstants()
3641 if (Subtarget.is64Bit() && ViaIntVT == MVT::i32) in lowerBuildVectorOfConstants()
3665 if (VT.isInteger() && EltBitSize < Subtarget.getELen() && in lowerBuildVectorOfConstants()
3668 (Sequence.size() * EltBitSize) <= Subtarget.getELen()) { in lowerBuildVectorOfConstants()
3680 (Subtarget.getRealMinVLen() >= ViaIntVT.getSizeInBits() * NumElts) ? in lowerBuildVectorOfConstants()
3698 if (Subtarget.is64Bit() && ViaIntVT == MVT::i32) in lowerBuildVectorOfConstants()
3705 (!Subtarget.is64Bit() && ViaIntVT == MVT::i64)) && in lowerBuildVectorOfConstants()
3711 getContainerForFixedLengthVector(DAG, ViaVecVT, Subtarget); in lowerBuildVectorOfConstants()
3716 Splat = convertFromScalableVector(ViaVecVT, Splat, DAG, Subtarget); in lowerBuildVectorOfConstants()
3732 (NumElts <= 4 || VT.getSizeInBits() > Subtarget.getRealMinVLen())) { in lowerBuildVectorOfConstants()
3738 Source, DAG, Subtarget); in lowerBuildVectorOfConstants()
3740 return convertFromScalableVector(VT, Res, DAG, Subtarget); in lowerBuildVectorOfConstants()
3744 if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget)) in lowerBuildVectorOfConstants()
3754 const RISCVSubtarget &Subtarget) { in lowerBUILD_VECTOR() argument
3760 return lowerBuildVectorOfConstants(Op, DAG, Subtarget); in lowerBUILD_VECTOR()
3762 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerBUILD_VECTOR()
3765 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerBUILD_VECTOR()
3767 MVT XLenVT = Subtarget.getXLenVT(); in lowerBUILD_VECTOR()
3794 if (auto Gather = matchSplatAsGather(Splat, VT, DL, DAG, Subtarget)) in lowerBUILD_VECTOR()
3802 return convertFromScalableVector(VT, Splat, DAG, Subtarget); in lowerBUILD_VECTOR()
3805 if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget)) in lowerBUILD_VECTOR()
3810 const unsigned MinVLen = Subtarget.getRealMinVLen(); in lowerBUILD_VECTOR()
3811 const unsigned MaxVLen = Subtarget.getRealMaxVLen(); in lowerBUILD_VECTOR()
3815 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerBUILD_VECTOR()
3817 MVT M1VT = getContainerForFixedLengthVector(DAG, OneRegVT, Subtarget); in lowerBUILD_VECTOR()
3831 SubBV = convertToScalableVector(M1VT, SubBV, DAG, Subtarget); in lowerBUILD_VECTOR()
3836 return convertFromScalableVector(VT, Vec, DAG, Subtarget); in lowerBUILD_VECTOR()
3885 VT.getVectorElementType().getSizeInBits() <= Subtarget.getFLen()) && in lowerBUILD_VECTOR()
3903 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerBUILD_VECTOR()
3909 const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT()); in lowerBUILD_VECTOR()
3910 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT), in lowerBUILD_VECTOR()
3917 V = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), V); in lowerBUILD_VECTOR()
3922 const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT()); in lowerBUILD_VECTOR()
3923 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT), in lowerBUILD_VECTOR()
3926 return convertFromScalableVector(VT, Vec, DAG, Subtarget); in lowerBUILD_VECTOR()
4000 const RISCVSubtarget &Subtarget) { in lowerScalarSplat() argument
4007 MVT XLenVT = Subtarget.getXLenVT(); in lowerScalarSplat()
4038 const RISCVSubtarget &Subtarget) { in lowerScalarInsert() argument
4041 const MVT XLenVT = Subtarget.getXLenVT(); in lowerScalarInsert()
4054 DAG, ExtractedContainerVT, Subtarget); in lowerScalarInsert()
4056 ExtractedVal, DAG, Subtarget); in lowerScalarInsert()
4076 VT, DL, DAG, Subtarget); in lowerScalarInsert()
4098 const RISCVSubtarget &Subtarget) { in isDeinterleaveShuffle() argument
4100 if (VT.getScalarSizeInBits() >= Subtarget.getELen()) in isDeinterleaveShuffle()
4142 int &OddSrc, const RISCVSubtarget &Subtarget) { in isInterleaveShuffle() argument
4144 if (VT.getScalarSizeInBits() >= Subtarget.getELen()) in isInterleaveShuffle()
4253 const RISCVSubtarget &Subtarget, in getDeinterleaveViaVNSRL() argument
4260 ContainerVT = getContainerForFixedLengthVector(DAG, ContainerVT, Subtarget); in getDeinterleaveViaVNSRL()
4266 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget); in getDeinterleaveViaVNSRL()
4269 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in getDeinterleaveViaVNSRL()
4286 DAG.getConstant(Shift, DL, Subtarget.getXLenVT()), VL); in getDeinterleaveViaVNSRL()
4294 Res = convertFromScalableVector(VT, Res, DAG, Subtarget); in getDeinterleaveViaVNSRL()
4313 const RISCVSubtarget &Subtarget, in lowerVECTOR_SHUFFLEAsVSlidedown() argument
4361 MVT XLenVT = Subtarget.getXLenVT(); in lowerVECTOR_SHUFFLEAsVSlidedown()
4363 MVT ContainerVT = getContainerForFixedLengthVector(DAG, SrcVT, Subtarget); in lowerVECTOR_SHUFFLEAsVSlidedown()
4364 auto [TrueMask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlidedown()
4366 getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT), in lowerVECTOR_SHUFFLEAsVSlidedown()
4367 convertToScalableVector(ContainerVT, Src, DAG, Subtarget), in lowerVECTOR_SHUFFLEAsVSlidedown()
4371 convertFromScalableVector(SrcVT, Slidedown, DAG, Subtarget), in lowerVECTOR_SHUFFLEAsVSlidedown()
4390 const RISCVSubtarget &Subtarget, in lowerVECTOR_SHUFFLEAsVSlideup() argument
4402 MVT XLenVT = Subtarget.getXLenVT(); in lowerVECTOR_SHUFFLEAsVSlideup()
4403 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVECTOR_SHUFFLEAsVSlideup()
4404 auto TrueMask = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).first; in lowerVECTOR_SHUFFLEAsVSlideup()
4413 InPlace = convertToScalableVector(ContainerVT, InPlace, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlideup()
4414 ToInsert = convertToScalableVector(ContainerVT, ToInsert, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlideup()
4424 Res = getVSlideup(DAG, Subtarget, DL, ContainerVT, InPlace, ToInsert, in lowerVECTOR_SHUFFLEAsVSlideup()
4426 return convertFromScalableVector(VT, Res, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlideup()
4434 const RISCVSubtarget &Subtarget, in lowerVECTOR_SHUFFLEAsVSlide1() argument
4468 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVECTOR_SHUFFLEAsVSlide1()
4469 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlide1()
4474 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Splat); in lowerVECTOR_SHUFFLEAsVSlide1()
4477 convertToScalableVector(ContainerVT, V2, DAG, Subtarget), in lowerVECTOR_SHUFFLEAsVSlide1()
4479 return convertFromScalableVector(VT, Vec, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlide1()
4487 const RISCVSubtarget &Subtarget) { in getWideningInterleave() argument
4492 VecContainerVT = getContainerForFixedLengthVector(DAG, VecVT, Subtarget); in getWideningInterleave()
4493 EvenV = convertToScalableVector(VecContainerVT, EvenV, DAG, Subtarget); in getWideningInterleave()
4494 OddV = convertToScalableVector(VecContainerVT, OddV, DAG, Subtarget); in getWideningInterleave()
4497 assert(VecVT.getScalarSizeInBits() < Subtarget.getELen()); in getWideningInterleave()
4508 WideContainerVT = getContainerForFixedLengthVector(DAG, WideVT, Subtarget); in getWideningInterleave()
4515 auto [Mask, VL] = getDefaultVLOps(VecVT, VecContainerVT, DL, DAG, Subtarget); in getWideningInterleave()
4519 if (Subtarget.hasStdExtZvbb()) { in getWideningInterleave()
4524 Subtarget.getXLenVT())); in getWideningInterleave()
4537 VecContainerVT, DL, DAG.getAllOnesConstant(DL, Subtarget.getXLenVT())); in getWideningInterleave()
4562 convertFromScalableVector(ResultVT, Interleaved, DAG, Subtarget); in getWideningInterleave()
4571 const RISCVSubtarget &Subtarget) { in lowerBitreverseShuffle() argument
4592 if (!Subtarget.getTargetLowering()->isOperationLegalOrCustom(ISD::BITREVERSE, in lowerBitreverseShuffle()
4594 !Subtarget.getTargetLowering()->isTypeLegal(ViaBitVT)) in lowerBitreverseShuffle()
4626 const RISCVSubtarget &Subtarget) { in lowerVECTOR_SHUFFLEAsRotate() argument
4640 if (!Subtarget.getTargetLowering()->isTypeLegal(RotateVT)) in lowerVECTOR_SHUFFLEAsRotate()
4662 const RISCVSubtarget &Subtarget) { in lowerShuffleViaVRegSplitting() argument
4672 const unsigned MinVLen = Subtarget.getRealMinVLen(); in lowerShuffleViaVRegSplitting()
4673 const unsigned MaxVLen = Subtarget.getRealMaxVLen(); in lowerShuffleViaVRegSplitting()
4707 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerShuffleViaVRegSplitting()
4709 MVT M1VT = getContainerForFixedLengthVector(DAG, OneRegVT, Subtarget); in lowerShuffleViaVRegSplitting()
4716 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget); in lowerShuffleViaVRegSplitting()
4717 V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget); in lowerShuffleViaVRegSplitting()
4726 SubVec = convertFromScalableVector(OneRegVT, SubVec, DAG, Subtarget); in lowerShuffleViaVRegSplitting()
4728 SubVec = convertToScalableVector(M1VT, SubVec, DAG, Subtarget); in lowerShuffleViaVRegSplitting()
4733 return convertFromScalableVector(VT, Vec, DAG, Subtarget); in lowerShuffleViaVRegSplitting()
4737 const RISCVSubtarget &Subtarget) { in lowerVECTOR_SHUFFLE() argument
4741 MVT XLenVT = Subtarget.getXLenVT(); in lowerVECTOR_SHUFFLE()
4749 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget)) in lowerVECTOR_SHUFFLE()
4751 if (SDValue V = lowerBitreverseShuffle(SVN, DAG, Subtarget)) in lowerVECTOR_SHUFFLE()
4764 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVECTOR_SHUFFLE()
4766 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
4809 return convertFromScalableVector(VT, NewLoad, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
4831 return convertFromScalableVector(VT, Splat, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
4834 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
4839 return convertFromScalableVector(VT, Gather, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
4845 if (SDValue V = lowerShuffleViaVRegSplitting(SVN, DAG, Subtarget)) in lowerVECTOR_SHUFFLE()
4851 lowerVECTOR_SHUFFLEAsVSlide1(DL, VT, V1, V2, Mask, Subtarget, DAG)) in lowerVECTOR_SHUFFLE()
4855 lowerVECTOR_SHUFFLEAsVSlidedown(DL, VT, V1, V2, Mask, Subtarget, DAG)) in lowerVECTOR_SHUFFLE()
4860 if (Subtarget.hasStdExtZvkb()) in lowerVECTOR_SHUFFLE()
4861 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget)) in lowerVECTOR_SHUFFLE()
4872 LoV = convertToScalableVector(ContainerVT, LoV, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
4876 HiV = convertToScalableVector(ContainerVT, HiV, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
4887 Res = getVSlidedown(DAG, Subtarget, DL, ContainerVT, Res, HiV, in lowerVECTOR_SHUFFLE()
4891 Res = getVSlideup(DAG, Subtarget, DL, ContainerVT, Res, LoV, in lowerVECTOR_SHUFFLE()
4895 return convertFromScalableVector(VT, Res, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
4900 if (isDeinterleaveShuffle(VT, ContainerVT, V1, V2, Mask, Subtarget)) { in lowerVECTOR_SHUFFLE()
4902 Subtarget, DAG); in lowerVECTOR_SHUFFLE()
4906 lowerVECTOR_SHUFFLEAsVSlideup(DL, VT, V1, V2, Mask, Subtarget, DAG)) in lowerVECTOR_SHUFFLE()
4912 if (isInterleaveShuffle(Mask, VT, EvenSrc, OddSrc, Subtarget)) { in lowerVECTOR_SHUFFLE()
4928 return getWideningInterleave(EvenV, OddV, DL, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
4967 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget)) in lowerVECTOR_SHUFFLE()
5030 (IndexVT.getSizeInBits() / Subtarget.getRealMinVLen()) > 1) { in lowerVECTOR_SHUFFLE()
5043 Subtarget); in lowerVECTOR_SHUFFLE()
5045 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
5057 convertToScalableVector(IndexContainerVT, LHSIndices, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
5067 V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
5071 convertToScalableVector(MaskContainerVT, SelectMask, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
5084 convertToScalableVector(IndexContainerVT, RHSIndices, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
5090 return convertFromScalableVector(VT, Gather, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
5110 isInterleaveShuffle(M, SVT, Dummy1, Dummy2, Subtarget); in isShuffleMaskLegal()
5129 Subtarget); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5169 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5172 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5174 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT()); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5180 FloatVal = convertFromScalableVector(FloatVT, FloatVal, DAG, Subtarget); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5293 const RISCVSubtarget &Subtarget) { in lowerConstant() argument
5306 if (!Subtarget.useConstantPoolForLargeInts()) in lowerConstant()
5309 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Imm, Subtarget); in lowerConstant()
5310 if (Seq.size() <= Subtarget.getMaxBuildIntsCost()) in lowerConstant()
5325 RISCVMatInt::generateTwoRegInstSeq(Imm, Subtarget, ShiftAmt, AddOpc); in lowerConstant()
5326 if (!SeqLo.empty() && (SeqLo.size() + 2) <= Subtarget.getMaxBuildIntsCost()) in lowerConstant()
5333 const RISCVSubtarget &Subtarget) { in LowerATOMIC_FENCE() argument
5340 if (Subtarget.hasStdExtZtso()) { in LowerATOMIC_FENCE()
5365 MVT XLenVT = Subtarget.getXLenVT(); in LowerIS_FPCLASS()
5399 auto [Mask, VL] = getDefaultScalableVLOps(VT0, DL, DAG, Subtarget); in LowerIS_FPCLASS()
5419 auto [Mask, VL] = getDefaultVLOps(VT0, ContainerVT0, DL, DAG, Subtarget); in LowerIS_FPCLASS()
5424 Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget); in LowerIS_FPCLASS()
5427 Op0 = convertToScalableVector(ContainerVT0, Op0, DAG, Subtarget); in LowerIS_FPCLASS()
5439 return convertFromScalableVector(VT, VMSEQ, DAG, Subtarget); in LowerIS_FPCLASS()
5451 return convertFromScalableVector(VT, VMSNE, DAG, Subtarget); in LowerIS_FPCLASS()
5464 const RISCVSubtarget &Subtarget) { in lowerFMAXIMUM_FMINIMUM() argument
5472 MVT XLenVT = Subtarget.getXLenVT(); in lowerFMAXIMUM_FMINIMUM()
5502 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerFMAXIMUM_FMINIMUM()
5503 X = convertToScalableVector(ContainerVT, X, DAG, Subtarget); in lowerFMAXIMUM_FMINIMUM()
5504 Y = convertToScalableVector(ContainerVT, Y, DAG, Subtarget); in lowerFMAXIMUM_FMINIMUM()
5512 Subtarget); in lowerFMAXIMUM_FMINIMUM()
5515 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerFMAXIMUM_FMINIMUM()
5543 Res = convertFromScalableVector(VT, Res, DAG, Subtarget); in lowerFMAXIMUM_FMINIMUM()
5841 return LowerATOMIC_FENCE(Op, DAG, Subtarget); in LowerOperation()
5853 return lowerConstant(Op, DAG, Subtarget); in LowerOperation()
5873 assert(Subtarget.hasStdExtZvkb()); in LowerOperation()
5876 assert(Subtarget.hasVendorXTHeadBb() && in LowerOperation()
5877 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) && in LowerOperation()
5888 MVT XLenVT = Subtarget.getXLenVT(); in LowerOperation()
5890 Subtarget.hasStdExtZfhminOrZhinxmin()) { in LowerOperation()
5896 Subtarget.hasStdExtZfbfmin()) { in LowerOperation()
5901 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.is64Bit() && in LowerOperation()
5902 Subtarget.hasStdExtFOrZfinx()) { in LowerOperation()
5909 Subtarget.hasStdExtZfa()) { in LowerOperation()
5970 assert(Subtarget.hasStdExtZvbb()); in LowerOperation()
5974 assert(Subtarget.hasStdExtZbkb() && "Unexpected custom legalization"); in LowerOperation()
6014 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in LowerOperation()
6015 Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Scalar); in LowerOperation()
6019 V = convertFromScalableVector(VT, V, DAG, Subtarget); in LowerOperation()
6023 MVT XLenVT = Subtarget.getXLenVT(); in LowerOperation()
6031 if (Subtarget.getRealMinVLen() < RISCV::RVVBitsPerBlock) in LowerOperation()
6060 if (Op.getValueType() == MVT::f16 && Subtarget.is64Bit() && in LowerOperation()
6074 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6075 !Subtarget.hasVInstructionsF16())) in LowerOperation()
6077 return lowerFMAXIMUM_FMINIMUM(Op, DAG, Subtarget); in LowerOperation()
6083 if (VT == MVT::f32 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) in LowerOperation()
6085 if (VT == MVT::f64 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) { in LowerOperation()
6100 if (VT == MVT::bf16 && Op0VT == MVT::f32 && Subtarget.hasStdExtZfbfmin()) in LowerOperation()
6102 if (VT == MVT::bf16 && Op0VT == MVT::f64 && Subtarget.hasStdExtZfbfmin() && in LowerOperation()
6103 Subtarget.hasStdExtDOrZdinx()) { in LowerOperation()
6121 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6122 !Subtarget.hasVInstructionsF16())) { in LowerOperation()
6140 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6141 !Subtarget.hasVInstructionsF16())) { in LowerOperation()
6278 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in LowerOperation()
6280 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget); in LowerOperation()
6284 SDValue SubVec = convertFromScalableVector(VT, Src, DAG, Subtarget); in LowerOperation()
6288 return convertFromScalableVector(VT, Src, DAG, Subtarget); in LowerOperation()
6292 return lowerFP_TO_INT_SAT(Op, DAG, Subtarget); in LowerOperation()
6296 assert(!Subtarget.isSoftFPABI() && "Unexpected custom legalization"); in LowerOperation()
6303 if (Subtarget.is64Bit() && !RV64LegalI32) in LowerOperation()
6308 assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalization"); in LowerOperation()
6314 SDValue Res = Subtarget.is64Bit() in LowerOperation()
6325 assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalisation"); in LowerOperation()
6332 if (Subtarget.is64Bit() && !RV64LegalI32) in LowerOperation()
6339 assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalisation"); in LowerOperation()
6342 SDValue Arg = Subtarget.is64Bit() in LowerOperation()
6358 return lowerFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget); in LowerOperation()
6361 return lowerVectorXRINT(Op, DAG, Subtarget); in LowerOperation()
6389 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6390 !Subtarget.hasVInstructionsF16())) in LowerOperation()
6402 DAG.getUNDEF(ContainerVT), DAG, Subtarget); in LowerOperation()
6419 return lowerBUILD_VECTOR(Op, DAG, Subtarget); in LowerOperation()
6422 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6423 !Subtarget.hasVInstructionsF16())) { in LowerOperation()
6440 return lowerVECTOR_SHUFFLE(Op, DAG, Subtarget); in LowerOperation()
6533 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6534 !Subtarget.hasVInstructionsF16())) in LowerOperation()
6560 assert(Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.is64Bit() && in LowerOperation()
6574 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6575 !Subtarget.hasVInstructionsF16())) in LowerOperation()
6596 if (Subtarget.hasStdExtZvbb()) in LowerOperation()
6604 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6605 !Subtarget.hasVInstructionsF16())) in LowerOperation()
6615 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6616 !Subtarget.hasVInstructionsF16())) in LowerOperation()
6629 return lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget); in LowerOperation()
6668 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6669 !Subtarget.hasVInstructionsF16())) in LowerOperation()
6692 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6693 !Subtarget.hasVInstructionsF16())) { in LowerOperation()
6711 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6712 !Subtarget.hasVInstructionsF16())) { in LowerOperation()
6727 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6728 !Subtarget.hasVInstructionsF16())) in LowerOperation()
6742 if (Subtarget.hasStdExtZvbb()) in LowerOperation()
6747 if (Subtarget.hasStdExtZvbb()) in LowerOperation()
6764 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6765 !Subtarget.hasVInstructionsF16())) in LowerOperation()
6767 return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget); in LowerOperation()
6771 (Subtarget.hasVInstructionsF16Minimal() && in LowerOperation()
6772 !Subtarget.hasVInstructionsF16())) in LowerOperation()
6774 return lowerFMAXIMUM_FMINIMUM(Op, DAG, Subtarget); in LowerOperation()
6814 if (isPositionIndependent() || Subtarget.allowTaggedGlobals()) { in getAddr()
6816 if (IsLocal && !Subtarget.allowTaggedGlobals()) in getAddr()
6910 MVT XLenVT = Subtarget.getXLenVT(); in getStaticTLSAddr()
7059 const RISCVSubtarget &Subtarget) { in combineSelectToBinOp() argument
7066 if (!Subtarget.hasConditionalMoveFusion()) { in combineSelectToBinOp()
7125 const RISCVSubtarget &Subtarget) { in foldBinOpIntoSelectIfProfitable() argument
7126 if (Subtarget.hasShortForwardBranchOpt()) in foldBinOpIntoSelectIfProfitable()
7188 MVT XLenVT = Subtarget.getXLenVT(); in lowerSELECT()
7201 if ((Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps()) && in lowerSELECT()
7224 if (SDValue V = combineSelectToBinOp(Op.getNode(), DAG, Subtarget)) in lowerSELECT()
7229 if (!Subtarget.hasConditionalMoveFusion()) in lowerSELECT()
7236 if (SDValue V = combineSelectToBinOp(Op.getNode(), DAG, Subtarget)) in lowerSELECT()
7244 DAG, Subtarget)) { in lowerSELECT()
7339 MVT XLenVT = Subtarget.getXLenVT(); in lowerBRCOND()
7376 const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo(); in lowerFRAMEADDR()
7381 int XLenInBytes = Subtarget.getXLen() / 8; in lowerFRAMEADDR()
7399 const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo(); in lowerRETURNADDR()
7403 MVT XLenVT = Subtarget.getXLenVT(); in lowerRETURNADDR()
7404 int XLenInBytes = Subtarget.getXLen() / 8; in lowerRETURNADDR()
7444 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT); in lowerShiftLeftParts()
7445 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT); in lowerShiftLeftParts()
7494 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT); in lowerShiftRightParts()
7495 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT); in lowerShiftRightParts()
7527 SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second; in lowerVectorMaskSplat()
7531 SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second; in lowerVectorMaskSplat()
7550 assert(!Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64 && in lowerSPLAT_VECTOR_PARTS()
7561 auto VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second; in lowerSPLAT_VECTOR_PARTS()
7567 Res = convertFromScalableVector(VecVT, Res, DAG, Subtarget); in lowerSPLAT_VECTOR_PARTS()
7595 SDValue CC = convertToScalableVector(I1ContainerVT, Src, DAG, Subtarget); in lowerVectorMaskExt()
7597 SDValue VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second; in lowerVectorMaskExt()
7599 MVT XLenVT = Subtarget.getXLenVT(); in lowerVectorMaskExt()
7611 return convertFromScalableVector(VecVT, Select, DAG, Subtarget); in lowerVectorMaskExt()
7631 convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget); in lowerFixedLengthVectorExtendToRVV()
7634 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerFixedLengthVectorExtendToRVV()
7638 return convertFromScalableVector(ExtVT, Ext, DAG, Subtarget); in lowerFixedLengthVectorExtendToRVV()
7664 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget); in lowerVectorMaskTruncLike()
7668 Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget); in lowerVectorMaskTruncLike()
7674 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerVectorMaskTruncLike()
7677 SDValue SplatOne = DAG.getConstant(1, DL, Subtarget.getXLenVT()); in lowerVectorMaskTruncLike()
7678 SDValue SplatZero = DAG.getConstant(0, DL, Subtarget.getXLenVT()); in lowerVectorMaskTruncLike()
7692 Trunc = convertFromScalableVector(MaskVT, Trunc, DAG, Subtarget); in lowerVectorMaskTruncLike()
7730 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget); in lowerVectorTruncLike()
7733 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerVectorTruncLike()
7740 getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget); in lowerVectorTruncLike()
7753 Result = convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVectorTruncLike()
7771 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget); in lowerStrictFPExtendOrRoundLike()
7774 auto [Mask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget); in lowerStrictFPExtendOrRoundLike()
7799 SDValue SubVec = convertFromScalableVector(VT, Res, DAG, Subtarget); in lowerStrictFPExtendOrRoundLike()
7841 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget); in lowerVectorFPExtendOrRoundLike()
7844 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerVectorFPExtendOrRoundLike()
7850 getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget); in lowerVectorFPExtendOrRoundLike()
7857 Src = convertFromScalableVector(VT, Src, DAG, Subtarget); in lowerVectorFPExtendOrRoundLike()
7870 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVectorFPExtendOrRoundLike()
7881 const RISCVSubtarget &Subtarget) { in getSmallestVTForIndex() argument
7884 const unsigned VectorBitsMin = Subtarget.getRealMinVLen(); in getSmallestVTForIndex()
7927 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerINSERT_VECTOR_ELT()
7939 DL, DAG, Subtarget)) { in lowerINSERT_VECTOR_ELT()
7947 const unsigned MinVLen = Subtarget.getRealMinVLen(); in lowerINSERT_VECTOR_ELT()
7948 const unsigned MaxVLen = Subtarget.getRealMaxVLen(); in lowerINSERT_VECTOR_ELT()
7967 MVT XLenVT = Subtarget.getXLenVT(); in lowerINSERT_VECTOR_ELT()
7969 bool IsLegalInsert = Subtarget.is64Bit() || Val.getValueType() != MVT::i64; in lowerINSERT_VECTOR_ELT()
7982 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerINSERT_VECTOR_ELT()
7999 return convertFromScalableVector(VecVT, Vec, DAG, Subtarget); in lowerINSERT_VECTOR_ELT()
8001 ValInVec = lowerScalarInsert(Val, VL, ContainerVT, DL, DAG, Subtarget); in lowerINSERT_VECTOR_ELT()
8012 getDefaultScalableVLOps(I32ContainerVT, DL, DAG, Subtarget).first; in lowerINSERT_VECTOR_ELT()
8035 return convertFromScalableVector(VecVT, ValInVec, DAG, Subtarget); in lowerINSERT_VECTOR_ELT()
8060 SDValue Slideup = getVSlideup(DAG, Subtarget, DL, ContainerVT, Vec, ValInVec, in lowerINSERT_VECTOR_ELT()
8068 return convertFromScalableVector(VecVT, Slideup, DAG, Subtarget); in lowerINSERT_VECTOR_ELT()
8082 MVT XLenVT = Subtarget.getXLenVT(); in lowerEXTRACT_VECTOR_ELT()
8090 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerEXTRACT_VECTOR_ELT()
8092 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerEXTRACT_VECTOR_ELT()
8106 unsigned MaxEEW = Subtarget.getELen(); in lowerEXTRACT_VECTOR_ELT()
8150 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerEXTRACT_VECTOR_ELT()
8157 const unsigned MinVLen = Subtarget.getRealMinVLen(); in lowerEXTRACT_VECTOR_ELT()
8158 const unsigned MaxVLen = Subtarget.getRealMaxVLen(); in lowerEXTRACT_VECTOR_ELT()
8185 getSmallestVTForIndex(ContainerVT, *MaxIdx, DL, DAG, Subtarget)) { in lowerEXTRACT_VECTOR_ELT()
8211 auto [Mask, VL] = getDefaultVLOps(1, ContainerVT, DL, DAG, Subtarget); in lowerEXTRACT_VECTOR_ELT()
8212 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, in lowerEXTRACT_VECTOR_ELT()
8229 const RISCVSubtarget &Subtarget) { in lowerVectorIntrinsicScalars() argument
8235 if (!Subtarget.hasVInstructions()) in lowerVectorIntrinsicScalars()
8255 MVT XLenVT = Subtarget.getXLenVT(); in lowerVectorIntrinsicScalars()
8316 RISCVTargetLowering::computeVLMAXBounds(VT, Subtarget); in lowerVectorIntrinsicScalars()
8417 const RISCVSubtarget &Subtarget) { in lowerGetVectorLength() argument
8418 MVT XLenVT = Subtarget.getXLenVT(); in lowerGetVectorLength()
8426 unsigned MinVF = RISCV::RVVBitsPerBlock / Subtarget.getELen(); in lowerGetVectorLength()
8455 const RISCVSubtarget &Subtarget = in getVCIXOperands() local
8466 DAG, V.getSimpleValueType(), Subtarget); in getVCIXOperands()
8467 Ops.push_back(convertToScalableVector(OpContainerVT, V, DAG, Subtarget)); in getVCIXOperands()
8475 const RISCVSubtarget &Subtarget) { in isValidEGW() argument
8476 return (Subtarget.getRealMinVLen() * in isValidEGW()
8485 MVT XLenVT = Subtarget.getXLenVT(); in LowerINTRINSIC_WO_CHAIN()
8514 if (RV64LegalI32 && Subtarget.is64Bit() && Op.getValueType() == MVT::i32) { in LowerINTRINSIC_WO_CHAIN()
8528 if (RV64LegalI32 && Subtarget.is64Bit() && Op.getValueType() == MVT::i32) { in LowerINTRINSIC_WO_CHAIN()
8548 if (RV64LegalI32 && Subtarget.is64Bit() && Op.getValueType() == MVT::i32) { in LowerINTRINSIC_WO_CHAIN()
8562 if (RV64LegalI32 && Subtarget.is64Bit() && Op.getValueType() == MVT::i32) { in LowerINTRINSIC_WO_CHAIN()
8580 return lowerGetVectorLength(Op.getNode(), DAG, Subtarget); in LowerINTRINSIC_WO_CHAIN()
8591 Subtarget); in LowerINTRINSIC_WO_CHAIN()
8660 if (!isValidEGW(4, Op.getSimpleValueType(), Subtarget) || in LowerINTRINSIC_WO_CHAIN()
8661 !isValidEGW(4, Op->getOperand(1).getSimpleValueType(), Subtarget) || in LowerINTRINSIC_WO_CHAIN()
8662 !isValidEGW(4, Op->getOperand(2).getSimpleValueType(), Subtarget)) in LowerINTRINSIC_WO_CHAIN()
8669 if (!isValidEGW(8, Op.getSimpleValueType(), Subtarget) || in LowerINTRINSIC_WO_CHAIN()
8670 !isValidEGW(8, Op->getOperand(1).getSimpleValueType(), Subtarget)) in LowerINTRINSIC_WO_CHAIN()
8679 !Subtarget.hasStdExtZvknhb()) in LowerINTRINSIC_WO_CHAIN()
8681 if (!isValidEGW(4, Op.getSimpleValueType(), Subtarget) || in LowerINTRINSIC_WO_CHAIN()
8682 !isValidEGW(4, Op->getOperand(1).getSimpleValueType(), Subtarget) || in LowerINTRINSIC_WO_CHAIN()
8683 !isValidEGW(4, Op->getOperand(2).getSimpleValueType(), Subtarget)) in LowerINTRINSIC_WO_CHAIN()
8716 NewNode = convertFromScalableVector(VT, NewNode, DAG, Subtarget); in LowerINTRINSIC_WO_CHAIN()
8727 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget); in LowerINTRINSIC_WO_CHAIN()
8738 MVT XLenVT = Subtarget.getXLenVT(); in LowerINTRINSIC_W_CHAIN()
8754 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in LowerINTRINSIC_W_CHAIN()
8755 PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget); in LowerINTRINSIC_W_CHAIN()
8760 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in LowerINTRINSIC_W_CHAIN()
8774 Subtarget); in LowerINTRINSIC_W_CHAIN()
8803 Result = convertFromScalableVector(VT, Result, DAG, Subtarget); in LowerINTRINSIC_W_CHAIN()
8821 MVT XLenVT = Subtarget.getXLenVT(); in LowerINTRINSIC_W_CHAIN()
8826 Subtarget); in LowerINTRINSIC_W_CHAIN()
8842 DAG, Subtarget)); in LowerINTRINSIC_W_CHAIN()
8877 convertFromScalableVector(VT, NewNode, DAG, Subtarget); in LowerINTRINSIC_W_CHAIN()
8891 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget); in LowerINTRINSIC_W_CHAIN()
8902 MVT XLenVT = Subtarget.getXLenVT(); in LowerINTRINSIC_VOID()
8914 Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget); in LowerINTRINSIC_VOID()
8919 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in LowerINTRINSIC_VOID()
8922 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in LowerINTRINSIC_VOID()
8957 MVT XLenVT = Subtarget.getXLenVT(); in LowerINTRINSIC_VOID()
8962 Subtarget); in LowerINTRINSIC_VOID()
8970 ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget)); in LowerINTRINSIC_VOID()
9046 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget); in LowerINTRINSIC_VOID()
9103 MVT XLenVT = Subtarget.getXLenVT(); in lowerVectorMaskVecReduction()
9108 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerVectorMaskVecReduction()
9117 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerVectorMaskVecReduction()
9183 const RISCVSubtarget &Subtarget) { in lowerReductionSeq() argument
9186 const MVT XLenVT = Subtarget.getXLenVT(); in lowerReductionSeq()
9197 DAG, Subtarget); in lowerReductionSeq()
9239 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerVECREDUCE()
9242 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerVECREDUCE()
9252 MVT XLenVT = Subtarget.getXLenVT(); in lowerVECREDUCE()
9257 Mask, VL, DL, DAG, Subtarget); in lowerVECREDUCE()
9265 const RISCVSubtarget &Subtarget) { in getRVVFPReductionOpAndOperands() argument
9283 MVT XLenVT = Subtarget.getXLenVT(); in getRVVFPReductionOpAndOperands()
9303 getRVVFPReductionOpAndOperands(Op, DAG, VecEltVT, Subtarget); in lowerFPVECREDUCE()
9309 VectorVal = convertToScalableVector(ContainerVT, VectorVal, DAG, Subtarget); in lowerFPVECREDUCE()
9312 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerFPVECREDUCE()
9314 VectorVal, Mask, VL, DL, DAG, Subtarget); in lowerFPVECREDUCE()
9333 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerVPREDUCE()
9339 Vec, Mask, VL, DL, DAG, Subtarget); in lowerVPREDUCE()
9350 MVT XLenVT = Subtarget.getXLenVT(); in lowerINSERT_SUBVECTOR()
9352 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo(); in lowerINSERT_SUBVECTOR()
9402 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerINSERT_SUBVECTOR()
9409 SubVec = convertFromScalableVector(VecVT, SubVec, DAG, Subtarget); in lowerINSERT_SUBVECTOR()
9417 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first; in lowerINSERT_SUBVECTOR()
9421 SDValue VL = getVLOp(EndIndex, ContainerVT, DL, DAG, Subtarget); in lowerINSERT_SUBVECTOR()
9435 SubVec = getVSlideup(DAG, Subtarget, DL, ContainerVT, Vec, SubVec, in lowerINSERT_SUBVECTOR()
9440 SubVec = convertFromScalableVector(VecVT, SubVec, DAG, Subtarget); in lowerINSERT_SUBVECTOR()
9489 auto [Mask, VL] = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget); in lowerINSERT_SUBVECTOR()
9505 SubVec = getVSlideup(DAG, Subtarget, DL, InterSubVT, AlignedExtract, SubVec, in lowerINSERT_SUBVECTOR()
9527 MVT XLenVT = Subtarget.getXLenVT(); in lowerEXTRACT_SUBVECTOR()
9529 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo(); in lowerEXTRACT_SUBVECTOR()
9582 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerEXTRACT_SUBVECTOR()
9588 getSmallestVTForIndex(ContainerVT, LastIdx, DL, DAG, Subtarget)) { in lowerEXTRACT_SUBVECTOR()
9595 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first; in lowerEXTRACT_SUBVECTOR()
9599 Subtarget); in lowerEXTRACT_SUBVECTOR()
9602 getVSlidedown(DAG, Subtarget, DL, ContainerVT, in lowerEXTRACT_SUBVECTOR()
9640 auto [Mask, VL] = getDefaultScalableVLOps(InterSubVT, DL, DAG, Subtarget); in lowerEXTRACT_SUBVECTOR()
9642 getVSlidedown(DAG, Subtarget, DL, InterSubVT, DAG.getUNDEF(InterSubVT), in lowerEXTRACT_SUBVECTOR()
9689 MVT XLenVT = Subtarget.getXLenVT(); in lowerVECTOR_DEINTERLEAVE()
9725 auto [Mask, VL] = getDefaultScalableVLOps(ConcatVT, DL, DAG, Subtarget); in lowerVECTOR_DEINTERLEAVE()
9730 if (VecVT.getScalarSizeInBits() < Subtarget.getELen()) { in lowerVECTOR_DEINTERLEAVE()
9732 getDeinterleaveViaVNSRL(DL, VecVT, Concat, true, Subtarget, DAG); in lowerVECTOR_DEINTERLEAVE()
9734 getDeinterleaveViaVNSRL(DL, VecVT, Concat, false, Subtarget, DAG); in lowerVECTOR_DEINTERLEAVE()
9774 MVT XLenVT = Subtarget.getXLenVT(); in lowerVECTOR_INTERLEAVE()
9799 if (VecVT.getScalarSizeInBits() < Subtarget.getELen()) { in lowerVECTOR_INTERLEAVE()
9801 DAG, Subtarget); in lowerVECTOR_INTERLEAVE()
9858 MVT XLenVT = Subtarget.getXLenVT(); in lowerSTEP_VECTOR()
9859 auto [Mask, VL] = getDefaultScalableVLOps(VT, DL, DAG, Subtarget); in lowerSTEP_VECTOR()
9871 VL, VT, DL, DAG, Subtarget); in lowerSTEP_VECTOR()
9895 unsigned VectorBitsMax = Subtarget.getRealMaxVLen(); in lowerVECTOR_REVERSE()
9931 MVT XLenVT = Subtarget.getXLenVT(); in lowerVECTOR_REVERSE()
9932 auto [Mask, VL] = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget); in lowerVECTOR_REVERSE()
9941 !Subtarget.is64Bit() && IntVT.getVectorElementType() == MVT::i64; in lowerVECTOR_REVERSE()
9962 MVT XLenVT = Subtarget.getXLenVT(); in lowerVECTOR_SPLICE()
9984 getVSlidedown(DAG, Subtarget, DL, VecVT, DAG.getUNDEF(VecVT), V1, in lowerVECTOR_SPLICE()
9986 return getVSlideup(DAG, Subtarget, DL, VecVT, SlideDown, V2, UpOffset, in lowerVECTOR_SPLICE()
10003 MVT XLenVT = Subtarget.getXLenVT(); in lowerFixedLengthVectorLoadToRVV()
10009 RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget); in lowerFixedLengthVectorLoadToRVV()
10015 SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget); in lowerFixedLengthVectorLoadToRVV()
10019 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, Subtarget); in lowerFixedLengthVectorLoadToRVV()
10034 SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget); in lowerFixedLengthVectorLoadToRVV()
10051 MVT XLenVT = Subtarget.getXLenVT(); in lowerFixedLengthVectorStoreToRVV()
10064 convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget); in lowerFixedLengthVectorStoreToRVV()
10070 RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget); in lowerFixedLengthVectorStoreToRVV()
10077 Subtarget); in lowerFixedLengthVectorStoreToRVV()
10112 MVT XLenVT = Subtarget.getXLenVT(); in lowerMaskedLoad()
10117 PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget); in lowerMaskedLoad()
10120 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerMaskedLoad()
10125 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerMaskedLoad()
10148 Result = convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerMaskedLoad()
10177 MVT XLenVT = Subtarget.getXLenVT(); in lowerMaskedStore()
10183 Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget); in lowerMaskedStore()
10186 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerMaskedStore()
10191 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerMaskedStore()
10215 convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget); in lowerFixedLengthVectorSetccToRVV()
10217 convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget); in lowerFixedLengthVectorSetccToRVV()
10221 DAG, Subtarget); in lowerFixedLengthVectorSetccToRVV()
10228 return convertFromScalableVector(VT, Cmp, DAG, Subtarget); in lowerFixedLengthVectorSetccToRVV()
10276 Op1 = convertToScalableVector(ContainerInVT, Op1, DAG, Subtarget); in lowerVectorStrictFSetcc()
10277 Op2 = convertToScalableVector(ContainerInVT, Op2, DAG, Subtarget); in lowerVectorStrictFSetcc()
10281 auto [Mask, VL] = getDefaultVLOps(InVT, ContainerInVT, DL, DAG, Subtarget); in lowerVectorStrictFSetcc()
10313 SDValue SubVec = convertFromScalableVector(VT, Res, DAG, Subtarget); in lowerVectorStrictFSetcc()
10331 X = convertToScalableVector(ContainerVT, X, DAG, Subtarget); in lowerABS()
10339 Subtarget); in lowerABS()
10342 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerABS()
10346 DAG.getConstant(0, DL, Subtarget.getXLenVT()), VL); in lowerABS()
10353 Max = convertFromScalableVector(VT, Max, DAG, Subtarget); in lowerABS()
10367 Mag = convertToScalableVector(ContainerVT, Mag, DAG, Subtarget); in lowerFixedLengthVectorFCOPYSIGNToRVV()
10368 Sign = convertToScalableVector(ContainerVT, Sign, DAG, Subtarget); in lowerFixedLengthVectorFCOPYSIGNToRVV()
10370 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerFixedLengthVectorFCOPYSIGNToRVV()
10375 return convertFromScalableVector(VT, CopySign, DAG, Subtarget); in lowerFixedLengthVectorFCOPYSIGNToRVV()
10387 convertToScalableVector(I1ContainerVT, Op.getOperand(0), DAG, Subtarget); in lowerFixedLengthVectorSelectToRVV()
10389 convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget); in lowerFixedLengthVectorSelectToRVV()
10391 convertToScalableVector(ContainerVT, Op.getOperand(2), DAG, Subtarget); in lowerFixedLengthVectorSelectToRVV()
10394 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerFixedLengthVectorSelectToRVV()
10399 return convertFromScalableVector(VT, Select, DAG, Subtarget); in lowerFixedLengthVectorSelectToRVV()
10425 Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget)); in lowerToScalableOp()
10429 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerToScalableOp()
10442 SDValue SubVec = convertFromScalableVector(VT, ScalableRes, DAG, Subtarget); in lowerToScalableOp()
10448 return convertFromScalableVector(VT, ScalableRes, DAG, Subtarget); in lowerToScalableOp()
10500 Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget)); in lowerVPOp()
10508 return convertFromScalableVector(VT, VPOp, DAG, Subtarget); in lowerVPOp()
10524 Src = convertToScalableVector(SrcVT, Src, DAG, Subtarget); in lowerVPExtMaskOp()
10527 MVT XLenVT = Subtarget.getXLenVT(); in lowerVPExtMaskOp()
10541 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPExtMaskOp()
10558 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget); in lowerVPSetCCMaskOp()
10559 Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget); in lowerVPSetCCMaskOp()
10620 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPSetCCMaskOp()
10638 Src = convertToScalableVector(SrcVT, Src, DAG, Subtarget); in lowerVPFPIntConvOp()
10640 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerVPFPIntConvOp()
10658 MVT XLenVT = Subtarget.getXLenVT(); in lowerVPFPIntConvOp()
10726 MVT XLenVT = Subtarget.getXLenVT(); in lowerVPFPIntConvOp()
10754 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPFPIntConvOp()
10769 const MVT XLenVT = Subtarget.getXLenVT(); in lowerVPSpliceExperimental()
10774 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget); in lowerVPSpliceExperimental()
10775 Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget); in lowerVPSpliceExperimental()
10777 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerVPSpliceExperimental()
10819 getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT), in lowerVPSpliceExperimental()
10821 SDValue Result = getVSlideup(DAG, Subtarget, DL, ContainerVT, SlideDown, Op2, in lowerVPSpliceExperimental()
10835 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPSpliceExperimental()
10843 MVT XLenVT = Subtarget.getXLenVT(); in lowerVPReverseExperimental()
10852 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget); in lowerVPReverseExperimental()
10854 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerVPReverseExperimental()
10877 unsigned VectorBitsMax = Subtarget.getRealMaxVLen(); in lowerVPReverseExperimental()
10914 Result = getVSlidedown(DAG, Subtarget, DL, GatherVT, in lowerVPReverseExperimental()
10928 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPReverseExperimental()
10956 return convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPReverseExperimental()
10974 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget); in lowerLogicVPOp()
10975 Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget); in lowerLogicVPOp()
10982 return convertFromScalableVector(VT, Val, DAG, Subtarget); in lowerLogicVPOp()
10988 MVT XLenVT = Subtarget.getXLenVT(); in lowerVPStridedLoad()
11010 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerVPStridedLoad()
11026 Result = convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerVPStridedLoad()
11034 MVT XLenVT = Subtarget.getXLenVT(); in lowerVPStridedStore()
11042 StoreVal = convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget); in lowerVPStridedStore()
11057 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerVPStridedStore()
11105 MVT XLenVT = Subtarget.getXLenVT(); in lowerMaskedGather()
11125 Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget); in lowerMaskedGather()
11129 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerMaskedGather()
11130 PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget); in lowerMaskedGather()
11135 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerMaskedGather()
11163 Result = convertFromScalableVector(VT, Result, DAG, Subtarget); in lowerMaskedGather()
11204 MVT XLenVT = Subtarget.getXLenVT(); in lowerMaskedScatter()
11224 Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget); in lowerMaskedScatter()
11225 Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget); in lowerMaskedScatter()
11229 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); in lowerMaskedScatter()
11234 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerMaskedScatter()
11257 const MVT XLenVT = Subtarget.getXLenVT(); in lowerGET_ROUNDING()
11288 const MVT XLenVT = Subtarget.getXLenVT(); in lowerSET_ROUNDING()
11322 bool isRISCV64 = Subtarget.is64Bit(); in lowerEH_DWARF_CFA()
11393 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && in ReplaceNodeResults()
11407 !Subtarget.hasStdExtZfhOrZhinx()) { in ReplaceNodeResults()
11425 !Subtarget.hasStdExtZfhOrZhinx()) || in ReplaceNodeResults()
11466 if (Op0.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) in ReplaceNodeResults()
11489 assert(!Subtarget.is64Bit() && in ReplaceNodeResults()
11519 unsigned XLen = Subtarget.getXLen(); in ReplaceNodeResults()
11534 MVT XLenVT = Subtarget.getXLenVT(); in ReplaceNodeResults()
11558 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && in ReplaceNodeResults()
11565 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && in ReplaceNodeResults()
11569 if (N->getOpcode() == ISD::SHL && Subtarget.hasStdExtZbs() && in ReplaceNodeResults()
11594 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && in ReplaceNodeResults()
11596 assert((Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() || in ReplaceNodeResults()
11597 Subtarget.hasVendorXTHeadBb()) && in ReplaceNodeResults()
11600 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb())) in ReplaceNodeResults()
11608 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && in ReplaceNodeResults()
11625 Subtarget.is64Bit() && Subtarget.hasStdExtM() && in ReplaceNodeResults()
11646 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && in ReplaceNodeResults()
11680 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && in ReplaceNodeResults()
11719 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && in ReplaceNodeResults()
11721 if (Subtarget.hasStdExtZbb()) { in ReplaceNodeResults()
11740 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && in ReplaceNodeResults()
11743 if (Subtarget.hasStdExtZbb()) { in ReplaceNodeResults()
11781 MVT XLenVT = Subtarget.getXLenVT(); in ReplaceNodeResults()
11783 Subtarget.hasStdExtZfhminOrZhinxmin()) { in ReplaceNodeResults()
11787 Subtarget.hasStdExtZfbfmin()) { in ReplaceNodeResults()
11790 } else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.is64Bit() && in ReplaceNodeResults()
11791 Subtarget.hasStdExtFOrZfinx()) { in ReplaceNodeResults()
11796 Subtarget.hasStdExtZfa()) { in ReplaceNodeResults()
11819 MVT XLenVT = Subtarget.getXLenVT(); in ReplaceNodeResults()
11820 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.is64Bit())) && in ReplaceNodeResults()
11822 assert(Subtarget.hasStdExtZbkb() && "Unexpected extension"); in ReplaceNodeResults()
11848 assert(!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 && in ReplaceNodeResults()
11856 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in ReplaceNodeResults()
11859 MVT XLenVT = Subtarget.getXLenVT(); in ReplaceNodeResults()
11862 auto [Mask, VL] = getDefaultVLOps(1, ContainerVT, DL, DAG, Subtarget); in ReplaceNodeResults()
11867 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, in ReplaceNodeResults()
11895 SDValue Res = lowerGetVectorLength(N, DAG, Subtarget); in ReplaceNodeResults()
11907 if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32) in ReplaceNodeResults()
11941 if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32) in ReplaceNodeResults()
11954 if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32) in ReplaceNodeResults()
11984 MVT XLenVT = Subtarget.getXLenVT(); in ReplaceNodeResults()
11988 Subtarget.getXLenVT(), N->getOperand(1)); in ReplaceNodeResults()
11993 assert(VT == MVT::i64 && !Subtarget.is64Bit() && in ReplaceNodeResults()
12005 auto [Mask, VL] = getDefaultVLOps(1, VecVT, DL, DAG, Subtarget); in ReplaceNodeResults()
12044 SDVTList VTs = DAG.getVTList(Subtarget.getXLenVT(), MVT::Other); in ReplaceNodeResults()
12089 const RISCVSubtarget &Subtarget) { in combineBinOpOfExtractToReduceTree() argument
12099 if (!Subtarget.hasVInstructions()) in combineBinOpOfExtractToReduceTree()
12137 if (SrcVecVT.getScalarSizeInBits() > Subtarget.getELen()) in combineBinOpOfExtractToReduceTree()
12185 const RISCVSubtarget &Subtarget) { in combineBinOpToReduce() argument
12271 ScalarV.getSimpleValueType(), DL, DAG, Subtarget); in combineBinOpToReduce()
12277 NewScalarV, DAG.getConstant(0, DL, Subtarget.getXLenVT())); in combineBinOpToReduce()
12291 const RISCVSubtarget &Subtarget) { in transformAddShlImm() argument
12293 if (!Subtarget.hasStdExtZba()) in transformAddShlImm()
12298 if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen()) in transformAddShlImm()
12348 const RISCVSubtarget &Subtarget) { in combineSelectAndUse() argument
12355 if (!Subtarget.hasConditionalMoveFusion()) { in combineSelectAndUse()
12357 if ((!Subtarget.hasStdExtZicond() && in combineSelectAndUse()
12358 !Subtarget.hasVendorXVentanaCondOps()) || in combineSelectAndUse()
12367 if (VT.getSizeInBits() > Subtarget.getXLen()) in combineSelectAndUse()
12413 const RISCVSubtarget &Subtarget) { in combineSelectAndUseCommutative() argument
12416 if (SDValue Result = combineSelectAndUse(N, N0, N1, DAG, AllOnes, Subtarget)) in combineSelectAndUseCommutative()
12418 if (SDValue Result = combineSelectAndUse(N, N1, N0, DAG, AllOnes, Subtarget)) in combineSelectAndUseCommutative()
12442 const RISCVSubtarget &Subtarget) { in transformAddImmMulImm() argument
12445 if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen()) in transformAddImmMulImm()
12516 const RISCVSubtarget &Subtarget) { in performADDCombine() argument
12519 if (SDValue V = transformAddImmMulImm(N, DAG, Subtarget)) in performADDCombine()
12521 if (SDValue V = transformAddShlImm(N, DAG, Subtarget)) in performADDCombine()
12523 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget)) in performADDCombine()
12525 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget)) in performADDCombine()
12530 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget); in performADDCombine()
12575 const RISCVSubtarget &Subtarget) { in performSUBCombine() argument
12596 return combineSelectAndUse(N, N1, N0, DAG, /*AllOnes*/ false, Subtarget); in performSUBCombine()
12647 const RISCVSubtarget &Subtarget) { in performTRUNCATECombine() argument
12656 if (!RV64LegalI32 && Subtarget.is64Bit() && Subtarget.hasStdExtZbs() && VT == MVT::i1 && in performTRUNCATECombine()
12674 const RISCVSubtarget &Subtarget) { in performANDCombine() argument
12683 if (!RV64LegalI32 && Subtarget.is64Bit() && Subtarget.hasStdExtZbs() && in performANDCombine()
12696 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget)) in performANDCombine()
12698 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget)) in performANDCombine()
12707 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ true, Subtarget); in performANDCombine()
12747 const RISCVSubtarget &Subtarget) { in performORCombine() argument
12750 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget)) in performORCombine()
12752 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget)) in performORCombine()
12770 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget); in performORCombine()
12774 const RISCVSubtarget &Subtarget) { in performXORCombine() argument
12781 if (!RV64LegalI32 && Subtarget.is64Bit() && Subtarget.hasStdExtZbs() && in performXORCombine()
12818 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget)) in performXORCombine()
12820 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget)) in performXORCombine()
12825 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget); in performXORCombine()
12941 const RISCVSubtarget &Subtarget) { in performSETCCCombine() argument
12947 if (OpVT != MVT::i64 || !Subtarget.is64Bit()) in performSETCCCombine()
12988 const RISCVSubtarget &Subtarget) { in performSIGN_EXTEND_INREGCombine() argument
13074 const RISCVSubtarget &Subtarget, in getOrCreateExtendedOp()
13089 auto [Mask, VL] = getMaskAndVL(Root, DAG, Subtarget); in getOrCreateExtendedOp()
13184 const RISCVSubtarget &Subtarget) { in fillUpExtensionSupport()
13204 if (!Subtarget.getTargetLowering()->isTypeLegal(NarrowElt.getValueType())) in fillUpExtensionSupport()
13215 std::tie(Mask, VL) = getDefaultScalableVLOps(VT, DL, DAG, Subtarget); in fillUpExtensionSupport()
13298 const RISCVSubtarget &Subtarget) { in NodeExtensionHelper()
13316 std::tie(Mask, VL) = getMaskAndVL(Root, DAG, Subtarget); in NodeExtensionHelper()
13325 fillUpExtensionSupport(Root, DAG, Subtarget); in NodeExtensionHelper()
13343 const RISCVSubtarget &Subtarget) { in getMaskAndVL()
13351 return getDefaultScalableVLOps(VT, DL, DAG, Subtarget); in getMaskAndVL()
13360 const RISCVSubtarget &Subtarget) const { in areVLAndMaskCompatible()
13361 auto [Mask, VL] = getMaskAndVL(Root, DAG, Subtarget); in areVLAndMaskCompatible()
13422 const RISCVSubtarget &Subtarget) const { in materialize()
13425 NodeExtensionHelper::getMaskAndVL(Root, DAG, Subtarget); in materialize()
13437 LHS.getOrCreateExtendedOp(Root, DAG, Subtarget, SExtLHS), in materialize()
13438 RHS.getOrCreateExtendedOp(Root, DAG, Subtarget, SExtRHS), in materialize()
13457 const RISCVSubtarget &Subtarget) { in canFoldToVWWithSameExtensionImpl() argument
13459 if (!LHS.areVLAndMaskCompatible(Root, DAG, Subtarget) || in canFoldToVWWithSameExtensionImpl()
13460 !RHS.areVLAndMaskCompatible(Root, DAG, Subtarget)) in canFoldToVWWithSameExtensionImpl()
13483 const RISCVSubtarget &Subtarget) { in canFoldToVWWithSameExtension() argument
13485 /*AllowZExt=*/true, DAG, Subtarget); in canFoldToVWWithSameExtension()
13495 const RISCVSubtarget &Subtarget) { in canFoldToVW_W() argument
13496 if (!RHS.areVLAndMaskCompatible(Root, DAG, Subtarget)) in canFoldToVW_W()
13521 const RISCVSubtarget &Subtarget) { in canFoldToVWWithSEXT() argument
13523 /*AllowZExt=*/false, DAG, Subtarget); in canFoldToVWWithSEXT()
13533 const RISCVSubtarget &Subtarget) { in canFoldToVWWithZEXT() argument
13535 /*AllowZExt=*/true, DAG, Subtarget); in canFoldToVWWithZEXT()
13545 const RISCVSubtarget &Subtarget) { in canFoldToVW_SU() argument
13549 if (!LHS.areVLAndMaskCompatible(Root, DAG, Subtarget) || in canFoldToVW_SU()
13550 !RHS.areVLAndMaskCompatible(Root, DAG, Subtarget)) in canFoldToVW_SU()
13602 const RISCVSubtarget &Subtarget) { in combineBinOp_VLToVWBinOp_VL() argument
13619 NodeExtensionHelper LHS(N, 0, DAG, Subtarget); in combineBinOp_VLToVWBinOp_VL()
13620 NodeExtensionHelper RHS(N, 1, DAG, Subtarget); in combineBinOp_VLToVWBinOp_VL()
13648 FoldingStrategy(N, LHS, RHS, DAG, Subtarget); in combineBinOp_VLToVWBinOp_VL()
13677 SDValue NewValue = Res.materialize(DAG, Subtarget); in combineBinOp_VLToVWBinOp_VL()
13707 const RISCVSubtarget &Subtarget = MF.getSubtarget<RISCVSubtarget>(); in tryMemPairCombine() local
13710 MVT XLenVT = Subtarget.getXLenVT(); in tryMemPairCombine()
13758 const RISCVSubtarget &Subtarget = MF.getSubtarget<RISCVSubtarget>(); in performMemPairCombine() local
13761 if (!Subtarget.hasVendorXTHeadMemPair()) in performMemPairCombine()
13843 const RISCVSubtarget &Subtarget) { in performFP_TO_INTCombine() argument
13846 MVT XLenVT = Subtarget.getXLenVT(); in performFP_TO_INTCombine()
13859 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh()) in performFP_TO_INTCombine()
13887 SrcContainerVT = getContainerForFixedLengthVector(DAG, SrcVT, Subtarget); in performFP_TO_INTCombine()
13888 XVal = convertToScalableVector(SrcContainerVT, XVal, DAG, Subtarget); in performFP_TO_INTCombine()
13890 getContainerForFixedLengthVector(DAG, ContainerVT, Subtarget); in performFP_TO_INTCombine()
13894 getDefaultVLOps(SrcVT, SrcContainerVT, DL, DAG, Subtarget); in performFP_TO_INTCombine()
13916 FpToInt = convertFromScalableVector(VT, FpToInt, DAG, Subtarget); in performFP_TO_INTCombine()
13946 const RISCVSubtarget &Subtarget) { in performFP_TO_INT_SATCombine() argument
13949 MVT XLenVT = Subtarget.getXLenVT(); in performFP_TO_INT_SATCombine()
13968 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh()) in performFP_TO_INT_SATCombine()
14008 const RISCVSubtarget &Subtarget) { in performBITREVERSECombine() argument
14009 assert(Subtarget.hasStdExtZbkb() && "Unexpected extension"); in performBITREVERSECombine()
14016 if (!VT.isScalarInteger() || VT.getSizeInBits() >= Subtarget.getXLen() || in performBITREVERSECombine()
14103 const RISCVSubtarget &Subtarget) { in performVFMADD_VLCombine() argument
14109 (Subtarget.hasVInstructionsF16Minimal() && in performVFMADD_VLCombine()
14110 !Subtarget.hasVInstructionsF16())) { in performVFMADD_VLCombine()
14165 const RISCVSubtarget &Subtarget) { in performVFMUL_VLCombine() argument
14168 (Subtarget.hasVInstructionsF16Minimal() && in performVFMUL_VLCombine()
14169 !Subtarget.hasVInstructionsF16())) { in performVFMUL_VLCombine()
14206 const RISCVSubtarget &Subtarget) { in performFADDSUB_VLCombine() argument
14209 (Subtarget.hasVInstructionsF16Minimal() && in performFADDSUB_VLCombine()
14210 !Subtarget.hasVInstructionsF16())) { in performFADDSUB_VLCombine()
14266 const RISCVSubtarget &Subtarget) { in performSRACombine() argument
14269 if (N->getValueType(0) != MVT::i64 || !Subtarget.is64Bit()) in performSRACombine()
14437 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) { in combine_CC() argument
14456 LHS.getOperand(0).getValueType() == Subtarget.getXLenVT()) { in combine_CC()
14639 const RISCVSubtarget &Subtarget) { in useInversedSetcc() argument
14652 if (Subtarget.hasStdExtZbs() && VT.isScalarInteger() && in useInversedSetcc()
14653 (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())) { in useInversedSetcc()
14670 const RISCVSubtarget &Subtarget) { in performSELECTCombine() argument
14674 if (SDValue V = useInversedSetcc(N, DAG, Subtarget)) in performSELECTCombine()
14677 if (Subtarget.hasConditionalMoveFusion()) in performSELECTCombine()
14693 const RISCVSubtarget &Subtarget, in performBUILD_VECTORCombine() argument
14745 const RISCVSubtarget &Subtarget, in performINSERT_VECTOR_ELTCombine() argument
14818 const RISCVSubtarget &Subtarget, in performCONCAT_VECTORSCombine() argument
14915 Subtarget.getXLenVT()); in performCONCAT_VECTORSCombine()
14948 const RISCVSubtarget &Subtarget) { in combineToVWMACC() argument
14987 const RISCVSubtarget &Subtarget) { in combineToVWMACC() argument
14991 Subtarget); in combineToVWMACC()
14994 }(N, DAG, Subtarget); in combineToVWMACC()
15132 const MVT XLenVT = Subtarget.getXLenVT(); in PerformDAGCombine()
15262 if (SDValue V = combineBinOp_VLToVWBinOp_VL(N, DCI, Subtarget)) in PerformDAGCombine()
15264 if (SDValue V = combineToVWMACC(N, DAG, Subtarget)) in PerformDAGCombine()
15266 return performADDCombine(N, DAG, Subtarget); in PerformDAGCombine()
15269 if (SDValue V = combineBinOp_VLToVWBinOp_VL(N, DCI, Subtarget)) in PerformDAGCombine()
15271 return performSUBCombine(N, DAG, Subtarget); in PerformDAGCombine()
15274 return performANDCombine(N, DCI, Subtarget); in PerformDAGCombine()
15276 return performORCombine(N, DCI, Subtarget); in PerformDAGCombine()
15278 return performXORCombine(N, DAG, Subtarget); in PerformDAGCombine()
15280 if (SDValue V = combineBinOp_VLToVWBinOp_VL(N, DCI, Subtarget)) in PerformDAGCombine()
15290 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget)) in PerformDAGCombine()
15292 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget)) in PerformDAGCombine()
15297 return performSETCCCombine(N, DAG, Subtarget); in PerformDAGCombine()
15299 return performSIGN_EXTEND_INREGCombine(N, DAG, Subtarget); in PerformDAGCombine()
15304 if (N->getValueType(0) == MVT::i64 && Subtarget.is64Bit()) { in PerformDAGCombine()
15373 return performTRUNCATECombine(N, DAG, Subtarget); in PerformDAGCombine()
15375 return performSELECTCombine(N, DAG, Subtarget); in PerformDAGCombine()
15411 if (!Subtarget.hasShortForwardBranchOpt() && isa<ConstantSDNode>(TrueV) && in PerformDAGCombine()
15425 DAG.getConstant(Subtarget.getXLen() - 1, DL, VT)); in PerformDAGCombine()
15436 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget)) in PerformDAGCombine()
15440 if (!Subtarget.hasConditionalMoveFusion()) { in PerformDAGCombine()
15503 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget)) in PerformDAGCombine()
15510 return performBITREVERSECombine(N, DAG, Subtarget); in PerformDAGCombine()
15513 return performFP_TO_INTCombine(N, DCI, Subtarget); in PerformDAGCombine()
15516 return performFP_TO_INT_SATCombine(N, DCI, Subtarget); in PerformDAGCombine()
15570 isSimpleVIDSequence(Index, Subtarget.getXLen()); in PerformDAGCombine()
15612 MGN->getMemOperand()->getBaseAlign(), Subtarget)) { in PerformDAGCombine()
15743 if (SDValue V = performSRACombine(N, DAG, Subtarget)) in PerformDAGCombine()
15755 DAG.getRegister(RISCV::X0, Subtarget.getXLenVT())); in PerformDAGCombine()
15761 if (SDValue V = combineBinOp_VLToVWBinOp_VL(N, DCI, Subtarget)) in PerformDAGCombine()
15763 return combineToVWMACC(N, DAG, Subtarget); in PerformDAGCombine()
15770 return combineBinOp_VLToVWBinOp_VL(N, DCI, Subtarget); in PerformDAGCombine()
15779 return performVFMADD_VLCombine(N, DAG, Subtarget); in PerformDAGCombine()
15781 return performVFMUL_VLCombine(N, DAG, Subtarget); in PerformDAGCombine()
15784 return performFADDSUB_VLCombine(N, DAG, Subtarget); in PerformDAGCombine()
15803 MemVT.getVectorElementType().bitsLE(Subtarget.getXLenVT()) && in PerformDAGCombine()
15805 MemVT.getSizeInBits() <= Subtarget.getXLen(); in PerformDAGCombine()
15829 if (RISCVMatInt::getIntMatCost(NewC, Subtarget.getXLen(), Subtarget, in PerformDAGCombine()
15880 DAG.getConstant(1, DL, Subtarget.getXLenVT()), MemVT, in PerformDAGCombine()
15894 DAG, Subtarget)) in PerformDAGCombine()
15899 if (SDValue V = performBUILD_VECTORCombine(N, DAG, Subtarget, *this)) in PerformDAGCombine()
15903 if (SDValue V = performCONCAT_VECTORSCombine(N, DAG, Subtarget, *this)) in PerformDAGCombine()
15907 if (SDValue V = performINSERT_VECTOR_ELTCombine(N, DAG, Subtarget, *this)) in PerformDAGCombine()
16071 assert(Subtarget.useRVVForFixedLengthVectors()); in PerformDAGCombine()
16109 return Subtarget.hasStdExtZbb() && in shouldTransformSignedTruncationCheck()
16110 ((KeptBits == 8 && XVT == MVT::i64 && !Subtarget.is64Bit()) || in shouldTransformSignedTruncationCheck()
16151 RISCVMatInt::getIntMatCost(C1Int, Ty.getSizeInBits(), Subtarget, in isDesirableToCommuteWithShift()
16154 ShiftedC1Int, Ty.getSizeInBits(), Subtarget, in isDesirableToCommuteWithShift()
16364 const unsigned MinVLenB = Subtarget.getRealMinVLen() / 8; in computeKnownBitsForTargetNode()
16365 const unsigned MaxVLenB = Subtarget.getRealMaxVLen() / 8; in computeKnownBitsForTargetNode()
16394 uint64_t MaxVL = Subtarget.getRealMaxVLen() / SEW; in computeKnownBitsForTargetNode()
16461 unsigned XLen = Subtarget.getXLen(); in ComputeNumSignBitsForTargetNode()
16486 assert(Subtarget.getXLen() == 64); in ComputeNumSignBitsForTargetNode()
16488 assert(Subtarget.hasStdExtA()); in ComputeNumSignBitsForTargetNode()
16607 const RISCVSubtarget &Subtarget) { in emitSplitF64Pseudo() argument
16646 const RISCVSubtarget &Subtarget) { in emitBuildPairF64Pseudo() argument
16702 const RISCVSubtarget &Subtarget) { in emitQuietFCMP() argument
16739 const RISCVSubtarget &Subtarget) { in EmitLoweredCascadedSelect() argument
16775 const RISCVInstrInfo &TII = *Subtarget.getInstrInfo(); in EmitLoweredCascadedSelect()
16841 const RISCVSubtarget &Subtarget) { in emitSelectPseudo() argument
16885 return EmitLoweredCascadedSelect(MI, *Next, BB, Subtarget); in emitSelectPseudo()
16914 const RISCVInstrInfo &TII = *Subtarget.getInstrInfo(); in emitSelectPseudo()
17031 const RISCVSubtarget &Subtarget) { in emitFROUND() argument
17070 assert(Subtarget.is64Bit() && "Expected 64-bit GPR."); in emitFROUND()
17079 assert(Subtarget.is64Bit() && "Expected 64-bit GPR."); in emitFROUND()
17114 const RISCVInstrInfo &TII = *Subtarget.getInstrInfo(); in emitFROUND()
17169 assert(!Subtarget.is64Bit() && in EmitInstrWithCustomInserter()
17180 return emitSelectPseudo(MI, BB, Subtarget); in EmitInstrWithCustomInserter()
17183 return emitBuildPairF64Pseudo(MI, BB, Subtarget); in EmitInstrWithCustomInserter()
17186 return emitSplitF64Pseudo(MI, BB, Subtarget); in EmitInstrWithCustomInserter()
17188 return emitQuietFCMP(MI, BB, RISCV::FLE_H, RISCV::FEQ_H, Subtarget); in EmitInstrWithCustomInserter()
17190 return emitQuietFCMP(MI, BB, RISCV::FLE_H_INX, RISCV::FEQ_H_INX, Subtarget); in EmitInstrWithCustomInserter()
17192 return emitQuietFCMP(MI, BB, RISCV::FLT_H, RISCV::FEQ_H, Subtarget); in EmitInstrWithCustomInserter()
17194 return emitQuietFCMP(MI, BB, RISCV::FLT_H_INX, RISCV::FEQ_H_INX, Subtarget); in EmitInstrWithCustomInserter()
17196 return emitQuietFCMP(MI, BB, RISCV::FLE_S, RISCV::FEQ_S, Subtarget); in EmitInstrWithCustomInserter()
17198 return emitQuietFCMP(MI, BB, RISCV::FLE_S_INX, RISCV::FEQ_S_INX, Subtarget); in EmitInstrWithCustomInserter()
17200 return emitQuietFCMP(MI, BB, RISCV::FLT_S, RISCV::FEQ_S, Subtarget); in EmitInstrWithCustomInserter()
17202 return emitQuietFCMP(MI, BB, RISCV::FLT_S_INX, RISCV::FEQ_S_INX, Subtarget); in EmitInstrWithCustomInserter()
17204 return emitQuietFCMP(MI, BB, RISCV::FLE_D, RISCV::FEQ_D, Subtarget); in EmitInstrWithCustomInserter()
17206 return emitQuietFCMP(MI, BB, RISCV::FLE_D_INX, RISCV::FEQ_D_INX, Subtarget); in EmitInstrWithCustomInserter()
17209 Subtarget); in EmitInstrWithCustomInserter()
17211 return emitQuietFCMP(MI, BB, RISCV::FLT_D, RISCV::FEQ_D, Subtarget); in EmitInstrWithCustomInserter()
17213 return emitQuietFCMP(MI, BB, RISCV::FLT_D_INX, RISCV::FEQ_D_INX, Subtarget); in EmitInstrWithCustomInserter()
17216 Subtarget); in EmitInstrWithCustomInserter()
17243 return emitFROUND(MI, BB, Subtarget); in EmitInstrWithCustomInserter()
17247 if (!Subtarget.is64Bit()) in EmitInstrWithCustomInserter()
17685 if (Subtarget.hasVInstructions()) in analyzeInputArgs()
17716 if (Subtarget.hasVInstructions()) in analyzeOutputArgs()
17739 const RISCVSubtarget &Subtarget) { in convertLocVTToValVT() argument
17745 Val = convertFromScalableVector(VA.getValVT(), Val, DAG, Subtarget); in convertLocVTToValVT()
17803 const RISCVSubtarget &Subtarget) { in convertValVTToLocVT() argument
17811 Val = convertToScalableVector(LocVT, Val, DAG, Subtarget); in convertValVTToLocVT()
17916 const RISCVSubtarget &Subtarget = TLI.getSubtarget(); in CC_RISCV_FastCC() local
17919 (Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZfhmin())) { in CC_RISCV_FastCC()
17931 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) { in CC_RISCV_FastCC()
17943 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) { in CC_RISCV_FastCC()
17957 (Subtarget.hasStdExtZhinx() || Subtarget.hasStdExtZhinxmin())) || in CC_RISCV_FastCC()
17958 (LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) || in CC_RISCV_FastCC()
17959 (LocVT == MVT::f64 && Subtarget.is64Bit() && in CC_RISCV_FastCC()
17960 Subtarget.hasStdExtZdinx())) { in CC_RISCV_FastCC()
18039 const RISCVSubtarget &Subtarget = in CC_RISCV_GHC() local
18042 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) { in CC_RISCV_GHC()
18054 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) { in CC_RISCV_GHC()
18066 if ((LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) || in CC_RISCV_GHC()
18067 (LocVT == MVT::f64 && Subtarget.hasStdExtZdinx() && in CC_RISCV_GHC()
18068 Subtarget.is64Bit())) { in CC_RISCV_GHC()
18096 if (Subtarget.isRVE()) in LowerFormalArguments()
18098 if (!Subtarget.hasStdExtFOrZfinx() || !Subtarget.hasStdExtDOrZdinx()) in LowerFormalArguments()
18118 MVT XLenVT = Subtarget.getXLenVT(); in LowerFormalArguments()
18119 unsigned XLenInBytes = Subtarget.getXLen() / 8; in LowerFormalArguments()
18180 ArrayRef<MCPhysReg> ArgRegs = RISCV::getArgGPRs(Subtarget.getTargetABI()); in LowerFormalArguments()
18287 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo(); in isEligibleForTailCallOptimization()
18325 MVT XLenVT = Subtarget.getXLenVT(); in LowerCall()
18334 if (Subtarget.isRVE()) in LowerCall()
18473 ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL, Subtarget); in LowerCall()
18545 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); in LowerCall()
18605 RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL, Subtarget); in LowerCall()
18620 if (Subtarget.hasVInstructions()) in CanLowerReturn()
18691 Val = convertValVTToLocVT(DAG, Val, VA, DL, Subtarget); in LowerReturn()
19061 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin()) in getRegForInlineAsmConstraint()
19063 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx()) in getRegForInlineAsmConstraint()
19065 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.is64Bit()) in getRegForInlineAsmConstraint()
19069 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) in getRegForInlineAsmConstraint()
19071 if (Subtarget.hasStdExtF() && VT == MVT::f32) in getRegForInlineAsmConstraint()
19073 if (Subtarget.hasStdExtD() && VT == MVT::f64) in getRegForInlineAsmConstraint()
19138 if (Subtarget.hasStdExtF()) { in getRegForInlineAsmConstraint()
19175 if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) { in getRegForInlineAsmConstraint()
19182 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) { in getRegForInlineAsmConstraint()
19190 if (Subtarget.hasVInstructions()) { in getRegForInlineAsmConstraint()
19281 DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT())); in LowerAsmOperandForConstraint()
19288 DAG.getTargetConstant(0, SDLoc(Op), Subtarget.getXLenVT())); in LowerAsmOperandForConstraint()
19296 DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT())); in LowerAsmOperandForConstraint()
19318 if (Subtarget.hasStdExtZtso()) { in emitLeadingFence()
19334 if (Subtarget.hasStdExtZtso()) { in emitTrailingFence()
19342 if (Subtarget.enableSeqCstTrailingFence() && isa<StoreInst>(Inst) && in emitTrailingFence()
19359 if (Subtarget.hasForcedAtomics()) in shouldExpandAtomicRMWInIR()
19438 unsigned XLen = Subtarget.getXLen(); in emitMaskedAtomicRMWIntrinsic()
19482 if (Subtarget.hasForcedAtomics()) in shouldExpandAtomicCmpXchgInIR()
19494 unsigned XLen = Subtarget.getXLen(); in emitMaskedAtomicCmpXchgIntrinsic()
19529 return Subtarget.hasStdExtZfhmin(); in shouldConvertFpToSat()
19531 return Subtarget.hasStdExtF(); in shouldConvertFpToSat()
19533 return Subtarget.hasStdExtD(); in shouldConvertFpToSat()
19542 if (Subtarget.is64Bit() && !isPositionIndependent() && in getJumpTableEncoding()
19552 assert(Subtarget.is64Bit() && !isPositionIndependent() && in LowerCustomJumpTableEntry()
19563 assert(Subtarget.getRealMinVLen() >= 64 && "zve32* unsupported"); in isVScaleKnownToBeAPowerOfTwo()
19574 if (!Subtarget.hasVendorXTHeadMemIdx()) in getIndexedAddressParts()
19663 return VT.isVector() ? Subtarget.hasVInstructionsF16() in isFMAFasterThanFMulAndFAdd()
19664 : Subtarget.hasStdExtZfhOrZhinx(); in isFMAFasterThanFMulAndFAdd()
19666 return Subtarget.hasStdExtFOrZfinx(); in isFMAFasterThanFMulAndFAdd()
19668 return Subtarget.hasStdExtDOrZdinx(); in isFMAFasterThanFMulAndFAdd()
19678 return Subtarget.hasStdExtZacas() ? ISD::ANY_EXTEND : ISD::SIGN_EXTEND; in getExtendForAtomicCmpSwapArg()
19694 if (Subtarget.isSoftFPABI() && (Type.isFloatingPoint() && !Type.isVector() && in shouldExtendTypeInLibCall()
19695 Type.getSizeInBits() < Subtarget.getXLen())) in shouldExtendTypeInLibCall()
19702 if (Subtarget.is64Bit() && Type == MVT::i32) in shouldSignExtendTypeInLibCall()
19712 Subtarget.hasStdExtM() || Subtarget.hasStdExtZmmul(); in decomposeMulByConstant()
19718 if (HasExtMOrZmmul && VT.getSizeInBits() > Subtarget.getXLen()) in decomposeMulByConstant()
19729 if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) && in decomposeMulByConstant()
19756 if (VT.getScalarSizeInBits() > Subtarget.getXLen()) in isMulAddWithConstProfitable()
19776 *Fast = Subtarget.hasFastUnalignedAccess() || in allowsMisalignedMemoryAccesses()
19777 Subtarget.enableUnalignedScalarMem(); in allowsMisalignedMemoryAccesses()
19778 return Subtarget.hasFastUnalignedAccess() || in allowsMisalignedMemoryAccesses()
19779 Subtarget.enableUnalignedScalarMem(); in allowsMisalignedMemoryAccesses()
19795 *Fast = Subtarget.hasFastUnalignedAccess(); in allowsMisalignedMemoryAccesses()
19796 return Subtarget.hasFastUnalignedAccess(); in allowsMisalignedMemoryAccesses()
19802 if (!Subtarget.hasVInstructions()) in getOptimalMemOpType()
19816 const unsigned MinVLenInBytes = Subtarget.getRealMinVLen()/8; in getOptimalMemOpType()
19826 MVT ELenVT = MVT::getIntegerVT(Subtarget.getELen()); in getOptimalMemOpType()
19831 if (PreferredVT != MVT::i8 && !Subtarget.hasFastUnalignedAccess()) { in getOptimalMemOpType()
19971 if (Subtarget.isTargetFuchsia()) in getIRStackGuard()
19993 if (!Subtarget.useRVVForFixedLengthVectors()) in isLegalInterleavedAccessType()
20012 if (!Subtarget.hasVInstructions()) in isLegalStridedLoadStore()
20016 if (DataType.isFixedLengthVector() && !Subtarget.useRVVForFixedLengthVectors()) in isLegalStridedLoadStore()
20023 if (!Subtarget.hasFastUnalignedAccess() && in isLegalStridedLoadStore()
20059 auto *XLenTy = Type::getIntNTy(LI->getContext(), Subtarget.getXLen()); in lowerInterleavedLoad()
20113 auto *XLenTy = Type::getIntNTy(SI->getContext(), Subtarget.getXLen()); in lowerInterleavedStore()
20160 Type *XLenTy = Type::getIntNTy(LI->getContext(), Subtarget.getXLen()); in lowerDeinterleaveIntrinsicToLoad()
20210 Type *XLenTy = Type::getIntNTy(SI->getContext(), Subtarget.getXLen()); in lowerInterleaveIntrinsicToStore()
20265 BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF); in getRegisterByName()
20266 if (!ReservedRegs.test(Reg) && !Subtarget.isRegisterReservedByUser(Reg)) in getRegisterByName()
20325 return isTypeLegal(VT) && Subtarget.hasStdExtZvbb(); in isCtpopFast()
20326 if (VT.isFixedLengthVector() && Subtarget.hasStdExtZvbb()) in isCtpopFast()
20328 return Subtarget.hasStdExtZbb() && in isCtpopFast()
20371 if (!Subtarget.hasShortForwardBranchOpt()) in BuildSDIVPow2()
20374 if (!(VT == MVT::i32 || (VT == MVT::i64 && Subtarget.is64Bit()))) in BuildSDIVPow2()
20385 if (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps()) in shouldFoldSelectWithSingleBitTest()
20386 return !Subtarget.hasStdExtZbs() && AndMask.ugt(1024); in shouldFoldSelectWithSingleBitTest()
20391 return Subtarget.getMinimumJumpTableEntries(); in getMinimumJumpTableEntries()