Lines Matching refs:N
84 void Select(SDNode *N) override;
90 bool hasNoVMLxHazardUse(SDNode *N) const;
93 bool SelectRegShifterOperand(SDValue N, SDValue &A,
96 bool SelectImmShifterOperand(SDValue N, SDValue &A,
98 bool SelectShiftRegShifterOperand(SDValue N, SDValue &A, SDValue &B, in SelectShiftRegShifterOperand() argument
101 return SelectRegShifterOperand(N, A, B, C, false); in SelectShiftRegShifterOperand()
103 bool SelectShiftImmShifterOperand(SDValue N, SDValue &A, SDValue &B) { in SelectShiftImmShifterOperand() argument
105 return SelectImmShifterOperand(N, A, B, false); in SelectShiftImmShifterOperand()
107 bool SelectShiftImmShifterOperandOneUse(SDValue N, SDValue &A, SDValue &B) { in SelectShiftImmShifterOperandOneUse() argument
108 if (!N.hasOneUse()) in SelectShiftImmShifterOperandOneUse()
110 return SelectImmShifterOperand(N, A, B, false); in SelectShiftImmShifterOperandOneUse()
113 bool SelectAddLikeOr(SDNode *Parent, SDValue N, SDValue &Out);
115 bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
116 bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
118 bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) { in SelectCMOVPred() argument
119 const ConstantSDNode *CN = cast<ConstantSDNode>(N); in SelectCMOVPred()
120 Pred = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(N), MVT::i32); in SelectCMOVPred()
125 bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
127 bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
129 bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
131 bool SelectAddrOffsetNone(SDValue N, SDValue &Base);
132 bool SelectAddrMode3(SDValue N, SDValue &Base,
134 bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
136 bool IsAddressingMode5(SDValue N, SDValue &Base, SDValue &Offset, bool FP16);
137 bool SelectAddrMode5(SDValue N, SDValue &Base, SDValue &Offset);
138 bool SelectAddrMode5FP16(SDValue N, SDValue &Base, SDValue &Offset);
139 bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align);
140 bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset);
142 bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label);
145 bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
146 bool SelectThumbAddrModeRRSext(SDValue N, SDValue &Base, SDValue &Offset);
147 bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base,
149 bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
151 bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
153 bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
155 bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
157 bool SelectTAddrModeImm7(SDValue N, SDValue &Base, SDValue &OffImm);
160 bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
162 bool SelectT2AddrModeImm8(SDValue N, SDValue &Base, SDValue &OffImm);
163 bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
165 bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
168 bool SelectT2AddrModeImm7Offset(SDNode *Op, SDValue N, SDValue &OffImm);
169 bool SelectT2AddrModeImm7Offset(SDNode *Op, SDValue N, SDValue &OffImm,
172 bool SelectT2AddrModeImm7(SDValue N, SDValue &Base, SDValue &OffImm);
173 bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
175 bool SelectT2AddrModeExclusive(SDValue N, SDValue &Base, SDValue &OffImm);
178 bool SelectImmediateInRange(SDValue N, SDValue &OffImm);
203 bool tryARMIndexedLoad(SDNode *N);
204 bool tryT1IndexedLoad(SDNode *N);
205 bool tryT2IndexedLoad(SDNode *N);
206 bool tryMVEIndexedLoad(SDNode *N);
207 bool tryFMULFixed(SDNode *N, SDLoc dl);
208 bool tryFP_TO_INT(SDNode *N, SDLoc dl);
209 bool transformFixedFloatingPointConversion(SDNode *N, SDNode *FMul,
217 void SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
225 void SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
232 void SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating,
250 void SelectMVE_WB(SDNode *N, const uint16_t *Opcodes, bool Predicated);
253 void SelectMVE_LongShift(SDNode *N, uint16_t Opcode, bool Immediate,
257 void SelectMVE_VADCSBC(SDNode *N, uint16_t OpcodeWithCarry,
262 void SelectMVE_VSHLC(SDNode *N, bool Predicated);
272 void SelectBaseMVE_VMLLDAV(SDNode *N, bool Predicated,
278 void SelectMVE_VMLLDAV(SDNode *N, bool Predicated, const uint16_t *OpcodesS,
282 void SelectMVE_VRMLLDAVH(SDNode *N, bool Predicated, const uint16_t *OpcodesS,
290 void SelectMVE_VLD(SDNode *N, unsigned NumVecs,
295 void SelectMVE_VxDUP(SDNode *N, const uint16_t *Opcodes,
304 void SelectCDE_CXxD(SDNode *N, uint16_t Opcode, size_t NumExtraOps,
310 void SelectVLDDup(SDNode *N, bool IsIntrinsic, bool isUpdating,
316 bool tryV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
318 bool tryInsertVectorElt(SDNode *N);
321 bool tryABSOp(SDNode *N);
323 bool tryReadRegister(SDNode *N);
324 bool tryWriteRegister(SDNode *N);
326 bool tryInlineAsm(SDNode *N);
328 void SelectCMPZ(SDNode *N, bool &SwitchEQNEToPLMI);
330 void SelectCMP_SWAP(SDNode *N);
358 bool canExtractShiftFromMul(const SDValue &N, unsigned MaxShift,
363 void replaceDAGValue(const SDValue &N, SDValue M);
373 static bool isInt32Immediate(SDNode *N, unsigned &Imm) { in INITIALIZE_PASS()
374 if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) { in INITIALIZE_PASS()
375 Imm = N->getAsZExtVal(); in INITIALIZE_PASS()
383 static bool isInt32Immediate(SDValue N, unsigned &Imm) { in isInt32Immediate() argument
384 return isInt32Immediate(N.getNode(), Imm); in isInt32Immediate()
390 static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) { in isOpcWithIntImmediate() argument
391 return N->getOpcode() == Opc && in isOpcWithIntImmediate()
392 isInt32Immediate(N->getOperand(1).getNode(), Imm); in isOpcWithIntImmediate()
423 for (SDNode &N : llvm::make_early_inc_range(CurDAG->allnodes())) { in PreprocessISelDAG()
424 if (N.getOpcode() != ISD::ADD) in PreprocessISelDAG()
436 SDValue N0 = N.getOperand(0); in PreprocessISelDAG()
437 SDValue N1 = N.getOperand(1); in PreprocessISelDAG()
494 CurDAG->UpdateNodeOperands(&N, N0, N1); in PreprocessISelDAG()
501 bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const { in hasNoVMLxHazardUse()
508 if (!N->hasOneUse()) in hasNoVMLxHazardUse()
511 SDNode *Use = *N->use_begin(); in hasNoVMLxHazardUse()
555 bool ARMDAGToDAGISel::canExtractShiftFromMul(const SDValue &N, in canExtractShiftFromMul() argument
559 assert(N.getOpcode() == ISD::MUL); in canExtractShiftFromMul()
564 if (!N.hasOneUse()) return false; in canExtractShiftFromMul()
566 ConstantSDNode *MulConst = dyn_cast<ConstantSDNode>(N.getOperand(1)); in canExtractShiftFromMul()
583 NewMulConst = CurDAG->getConstant(NewMulConstVal, SDLoc(N), MVT::i32); in canExtractShiftFromMul()
589 void ARMDAGToDAGISel::replaceDAGValue(const SDValue &N, SDValue M) { in replaceDAGValue() argument
590 CurDAG->RepositionNode(N.getNode()->getIterator(), M.getNode()); in replaceDAGValue()
591 ReplaceUses(N, M); in replaceDAGValue()
594 bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N, in SelectImmShifterOperand() argument
603 if (N.getOpcode() == ISD::MUL) { in SelectImmShifterOperand()
606 if (canExtractShiftFromMul(N, 31, PowerOfTwo, NewMulConst)) { in SelectImmShifterOperand()
607 HandleSDNode Handle(N); in SelectImmShifterOperand()
608 SDLoc Loc(N); in SelectImmShifterOperand()
609 replaceDAGValue(N.getOperand(1), NewMulConst); in SelectImmShifterOperand()
617 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); in SelectImmShifterOperand()
623 BaseReg = N.getOperand(0); in SelectImmShifterOperand()
625 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); in SelectImmShifterOperand()
629 SDLoc(N), MVT::i32); in SelectImmShifterOperand()
633 bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N, in SelectRegShifterOperand() argument
641 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); in SelectRegShifterOperand()
647 BaseReg = N.getOperand(0); in SelectRegShifterOperand()
649 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); in SelectRegShifterOperand()
652 ShReg = N.getOperand(1); in SelectRegShifterOperand()
653 if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal)) in SelectRegShifterOperand()
656 SDLoc(N), MVT::i32); in SelectRegShifterOperand()
662 bool ARMDAGToDAGISel::SelectAddLikeOr(SDNode *Parent, SDValue N, SDValue &Out) { in SelectAddLikeOr() argument
664 Out = N; in SelectAddLikeOr()
665 return CurDAG->haveNoCommonBitsSet(N, Parent->getOperand(1)); in SelectAddLikeOr()
669 bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N, in SelectAddrModeImm12() argument
675 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && in SelectAddrModeImm12()
676 !CurDAG->isBaseWithConstantOffset(N)) { in SelectAddrModeImm12()
677 if (N.getOpcode() == ISD::FrameIndex) { in SelectAddrModeImm12()
679 int FI = cast<FrameIndexSDNode>(N)->getIndex(); in SelectAddrModeImm12()
682 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectAddrModeImm12()
686 if (N.getOpcode() == ARMISD::Wrapper && in SelectAddrModeImm12()
687 N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress && in SelectAddrModeImm12()
688 N.getOperand(0).getOpcode() != ISD::TargetExternalSymbol && in SelectAddrModeImm12()
689 N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress) { in SelectAddrModeImm12()
690 Base = N.getOperand(0); in SelectAddrModeImm12()
692 Base = N; in SelectAddrModeImm12()
693 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectAddrModeImm12()
697 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { in SelectAddrModeImm12()
699 if (N.getOpcode() == ISD::SUB) in SelectAddrModeImm12()
703 Base = N.getOperand(0); in SelectAddrModeImm12()
709 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32); in SelectAddrModeImm12()
715 Base = N; in SelectAddrModeImm12()
716 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectAddrModeImm12()
722 bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, in SelectLdStSOReg() argument
724 if (N.getOpcode() == ISD::MUL && in SelectLdStSOReg()
725 ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) { in SelectLdStSOReg()
726 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { in SelectLdStSOReg()
738 Base = Offset = N.getOperand(0); in SelectLdStSOReg()
741 SDLoc(N), MVT::i32); in SelectLdStSOReg()
748 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && in SelectLdStSOReg()
750 !CurDAG->isBaseWithConstantOffset(N)) in SelectLdStSOReg()
754 if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) { in SelectLdStSOReg()
756 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1, in SelectLdStSOReg()
762 ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::SUB ? ARM_AM::sub:ARM_AM::add; in SelectLdStSOReg()
764 ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode()); in SelectLdStSOReg()
767 Base = N.getOperand(0); in SelectLdStSOReg()
768 Offset = N.getOperand(1); in SelectLdStSOReg()
774 dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) { in SelectLdStSOReg()
777 Offset = N.getOperand(1).getOperand(0); in SelectLdStSOReg()
788 if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift && in SelectLdStSOReg()
790 N.getOperand(0).hasOneUse())) { in SelectLdStSOReg()
791 ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode()); in SelectLdStSOReg()
796 dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) { in SelectLdStSOReg()
798 if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) { in SelectLdStSOReg()
799 Offset = N.getOperand(0).getOperand(0); in SelectLdStSOReg()
800 Base = N.getOperand(1); in SelectLdStSOReg()
813 if (Offset.getOpcode() == ISD::MUL && N.hasOneUse()) { in SelectLdStSOReg()
826 SDLoc(N), MVT::i32); in SelectLdStSOReg()
830 bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N, in SelectAddrMode2OffsetReg() argument
839 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) in SelectAddrMode2OffsetReg()
842 Offset = N; in SelectAddrMode2OffsetReg()
843 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); in SelectAddrMode2OffsetReg()
848 if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) { in SelectAddrMode2OffsetReg()
850 if (isShifterOpProfitable(N, ShOpcVal, ShAmt)) in SelectAddrMode2OffsetReg()
851 Offset = N.getOperand(0); in SelectAddrMode2OffsetReg()
862 SDLoc(N), MVT::i32); in SelectAddrMode2OffsetReg()
866 bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N, in SelectAddrMode2OffsetImmPre() argument
875 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits. in SelectAddrMode2OffsetImmPre()
886 bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N, in SelectAddrMode2OffsetImm() argument
895 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits. in SelectAddrMode2OffsetImm()
906 bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) { in SelectAddrOffsetNone() argument
907 Base = N; in SelectAddrOffsetNone()
911 bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N, in SelectAddrMode3() argument
914 if (N.getOpcode() == ISD::SUB) { in SelectAddrMode3()
916 Base = N.getOperand(0); in SelectAddrMode3()
917 Offset = N.getOperand(1); in SelectAddrMode3()
918 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0), SDLoc(N), in SelectAddrMode3()
923 if (!CurDAG->isBaseWithConstantOffset(N)) { in SelectAddrMode3()
924 Base = N; in SelectAddrMode3()
925 if (N.getOpcode() == ISD::FrameIndex) { in SelectAddrMode3()
926 int FI = cast<FrameIndexSDNode>(N)->getIndex(); in SelectAddrMode3()
931 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N), in SelectAddrMode3()
938 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1, in SelectAddrMode3()
940 Base = N.getOperand(0); in SelectAddrMode3()
953 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC), SDLoc(N), in SelectAddrMode3()
958 Base = N.getOperand(0); in SelectAddrMode3()
959 Offset = N.getOperand(1); in SelectAddrMode3()
960 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N), in SelectAddrMode3()
965 bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N, in SelectAddrMode3Offset() argument
974 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits. in SelectAddrMode3Offset()
981 Offset = N; in SelectAddrMode3Offset()
987 bool ARMDAGToDAGISel::IsAddressingMode5(SDValue N, SDValue &Base, SDValue &Offset, in IsAddressingMode5() argument
989 if (!CurDAG->isBaseWithConstantOffset(N)) { in IsAddressingMode5()
990 Base = N; in IsAddressingMode5()
991 if (N.getOpcode() == ISD::FrameIndex) { in IsAddressingMode5()
992 int FI = cast<FrameIndexSDNode>(N)->getIndex(); in IsAddressingMode5()
995 } else if (N.getOpcode() == ARMISD::Wrapper && in IsAddressingMode5()
996 N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress && in IsAddressingMode5()
997 N.getOperand(0).getOpcode() != ISD::TargetExternalSymbol && in IsAddressingMode5()
998 N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress) { in IsAddressingMode5()
999 Base = N.getOperand(0); in IsAddressingMode5()
1002 SDLoc(N), MVT::i32); in IsAddressingMode5()
1010 if (isScaledConstantInRange(N.getOperand(1), Scale, -255, 256, RHSC)) { in IsAddressingMode5()
1011 Base = N.getOperand(0); in IsAddressingMode5()
1026 SDLoc(N), MVT::i32); in IsAddressingMode5()
1029 SDLoc(N), MVT::i32); in IsAddressingMode5()
1034 Base = N; in IsAddressingMode5()
1038 SDLoc(N), MVT::i32); in IsAddressingMode5()
1041 SDLoc(N), MVT::i32); in IsAddressingMode5()
1046 bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N, in SelectAddrMode5() argument
1048 return IsAddressingMode5(N, Base, Offset, /*FP16=*/ false); in SelectAddrMode5()
1051 bool ARMDAGToDAGISel::SelectAddrMode5FP16(SDValue N, in SelectAddrMode5FP16() argument
1053 return IsAddressingMode5(N, Base, Offset, /*FP16=*/ true); in SelectAddrMode5FP16()
1056 bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr, in SelectAddrMode6() argument
1058 Addr = N; in SelectAddrMode6()
1081 Align = CurDAG->getTargetConstant(Alignment, SDLoc(N), MVT::i32); in SelectAddrMode6()
1085 bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N, in SelectAddrMode6Offset() argument
1091 Offset = N; in SelectAddrMode6Offset()
1092 if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) { in SelectAddrMode6Offset()
1099 bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N, in SelectAddrModePC() argument
1101 if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) { in SelectAddrModePC()
1102 Offset = N.getOperand(0); in SelectAddrModePC()
1103 SDValue N1 = N.getOperand(1); in SelectAddrModePC()
1104 Label = CurDAG->getTargetConstant(N1->getAsZExtVal(), SDLoc(N), MVT::i32); in SelectAddrModePC()
1116 static bool shouldUseZeroOffsetLdSt(SDValue N) { in shouldUseZeroOffsetLdSt() argument
1120 if (N.getOpcode() != ISD::ADD) in shouldUseZeroOffsetLdSt()
1124 if (auto C = dyn_cast<ConstantSDNode>(N.getOperand(1))) in shouldUseZeroOffsetLdSt()
1130 bool ARMDAGToDAGISel::SelectThumbAddrModeRRSext(SDValue N, SDValue &Base, in SelectThumbAddrModeRRSext() argument
1132 if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) { in SelectThumbAddrModeRRSext()
1133 if (!isNullConstant(N)) in SelectThumbAddrModeRRSext()
1136 Base = Offset = N; in SelectThumbAddrModeRRSext()
1140 Base = N.getOperand(0); in SelectThumbAddrModeRRSext()
1141 Offset = N.getOperand(1); in SelectThumbAddrModeRRSext()
1145 bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N, SDValue &Base, in SelectThumbAddrModeRR() argument
1147 if (shouldUseZeroOffsetLdSt(N)) in SelectThumbAddrModeRR()
1149 return SelectThumbAddrModeRRSext(N, Base, Offset); in SelectThumbAddrModeRR()
1153 ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, in SelectThumbAddrModeImm5S() argument
1155 if (shouldUseZeroOffsetLdSt(N)) { in SelectThumbAddrModeImm5S()
1156 Base = N; in SelectThumbAddrModeImm5S()
1157 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectThumbAddrModeImm5S()
1161 if (!CurDAG->isBaseWithConstantOffset(N)) { in SelectThumbAddrModeImm5S()
1162 if (N.getOpcode() == ISD::ADD) { in SelectThumbAddrModeImm5S()
1164 } else if (N.getOpcode() == ARMISD::Wrapper && in SelectThumbAddrModeImm5S()
1165 N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress && in SelectThumbAddrModeImm5S()
1166 N.getOperand(0).getOpcode() != ISD::TargetExternalSymbol && in SelectThumbAddrModeImm5S()
1167 N.getOperand(0).getOpcode() != ISD::TargetConstantPool && in SelectThumbAddrModeImm5S()
1168 N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress) { in SelectThumbAddrModeImm5S()
1169 Base = N.getOperand(0); in SelectThumbAddrModeImm5S()
1171 Base = N; in SelectThumbAddrModeImm5S()
1174 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectThumbAddrModeImm5S()
1180 if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) { in SelectThumbAddrModeImm5S()
1181 Base = N.getOperand(0); in SelectThumbAddrModeImm5S()
1182 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32); in SelectThumbAddrModeImm5S()
1191 ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base, in SelectThumbAddrModeImm5S4() argument
1193 return SelectThumbAddrModeImm5S(N, 4, Base, OffImm); in SelectThumbAddrModeImm5S4()
1197 ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base, in SelectThumbAddrModeImm5S2() argument
1199 return SelectThumbAddrModeImm5S(N, 2, Base, OffImm); in SelectThumbAddrModeImm5S2()
1203 ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base, in SelectThumbAddrModeImm5S1() argument
1205 return SelectThumbAddrModeImm5S(N, 1, Base, OffImm); in SelectThumbAddrModeImm5S1()
1208 bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N, in SelectThumbAddrModeSP() argument
1210 if (N.getOpcode() == ISD::FrameIndex) { in SelectThumbAddrModeSP()
1211 int FI = cast<FrameIndexSDNode>(N)->getIndex(); in SelectThumbAddrModeSP()
1219 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectThumbAddrModeSP()
1223 if (!CurDAG->isBaseWithConstantOffset(N)) in SelectThumbAddrModeSP()
1226 if (N.getOperand(0).getOpcode() == ISD::FrameIndex) { in SelectThumbAddrModeSP()
1229 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) { in SelectThumbAddrModeSP()
1230 Base = N.getOperand(0); in SelectThumbAddrModeSP()
1244 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32); in SelectThumbAddrModeSP()
1255 bool ARMDAGToDAGISel::SelectTAddrModeImm7(SDValue N, SDValue &Base, in SelectTAddrModeImm7() argument
1257 if (N.getOpcode() == ISD::SUB || CurDAG->isBaseWithConstantOffset(N)) { in SelectTAddrModeImm7()
1259 if (isScaledConstantInRange(N.getOperand(1), 1 << Shift, -0x7f, 0x80, in SelectTAddrModeImm7()
1261 Base = N.getOperand(0); in SelectTAddrModeImm7()
1262 if (N.getOpcode() == ISD::SUB) in SelectTAddrModeImm7()
1265 CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32); in SelectTAddrModeImm7()
1271 Base = N; in SelectTAddrModeImm7()
1272 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectTAddrModeImm7()
1282 bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N, in SelectT2AddrModeImm12() argument
1287 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && in SelectT2AddrModeImm12()
1288 !CurDAG->isBaseWithConstantOffset(N)) { in SelectT2AddrModeImm12()
1289 if (N.getOpcode() == ISD::FrameIndex) { in SelectT2AddrModeImm12()
1291 int FI = cast<FrameIndexSDNode>(N)->getIndex(); in SelectT2AddrModeImm12()
1294 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectT2AddrModeImm12()
1298 if (N.getOpcode() == ARMISD::Wrapper && in SelectT2AddrModeImm12()
1299 N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress && in SelectT2AddrModeImm12()
1300 N.getOperand(0).getOpcode() != ISD::TargetExternalSymbol && in SelectT2AddrModeImm12()
1301 N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress) { in SelectT2AddrModeImm12()
1302 Base = N.getOperand(0); in SelectT2AddrModeImm12()
1306 Base = N; in SelectT2AddrModeImm12()
1307 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectT2AddrModeImm12()
1311 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { in SelectT2AddrModeImm12()
1312 if (SelectT2AddrModeImm8(N, Base, OffImm)) in SelectT2AddrModeImm12()
1317 if (N.getOpcode() == ISD::SUB) in SelectT2AddrModeImm12()
1321 Base = N.getOperand(0); in SelectT2AddrModeImm12()
1327 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32); in SelectT2AddrModeImm12()
1333 Base = N; in SelectT2AddrModeImm12()
1334 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectT2AddrModeImm12()
1339 bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N, SDValue &Base, in SelectT2AddrModeImm8() argument
1341 if (N.getOpcode() == ISD::SUB || CurDAG->isBaseWithConstantOffset(N)) { in SelectT2AddrModeImm8()
1343 if (isScaledConstantInRange(N.getOperand(1), 1 << Shift, -255, 256, RHSC)) { in SelectT2AddrModeImm8()
1344 Base = N.getOperand(0); in SelectT2AddrModeImm8()
1351 if (N.getOpcode() == ISD::SUB) in SelectT2AddrModeImm8()
1354 CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32); in SelectT2AddrModeImm8()
1360 Base = N; in SelectT2AddrModeImm8()
1361 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectT2AddrModeImm8()
1365 bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N, in SelectT2AddrModeImm8() argument
1368 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && in SelectT2AddrModeImm8()
1369 !CurDAG->isBaseWithConstantOffset(N)) in SelectT2AddrModeImm8()
1372 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { in SelectT2AddrModeImm8()
1374 if (N.getOpcode() == ISD::SUB) in SelectT2AddrModeImm8()
1378 Base = N.getOperand(0); in SelectT2AddrModeImm8()
1384 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32); in SelectT2AddrModeImm8()
1392 bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N, in SelectT2AddrModeImm8Offset() argument
1399 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits. in SelectT2AddrModeImm8Offset()
1401 ? CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32) in SelectT2AddrModeImm8Offset()
1402 : CurDAG->getTargetConstant(-RHSC, SDLoc(N), MVT::i32); in SelectT2AddrModeImm8Offset()
1410 bool ARMDAGToDAGISel::SelectT2AddrModeImm7(SDValue N, SDValue &Base, in SelectT2AddrModeImm7() argument
1412 if (N.getOpcode() == ISD::SUB || CurDAG->isBaseWithConstantOffset(N)) { in SelectT2AddrModeImm7()
1414 if (isScaledConstantInRange(N.getOperand(1), 1 << Shift, -0x7f, 0x80, in SelectT2AddrModeImm7()
1416 Base = N.getOperand(0); in SelectT2AddrModeImm7()
1423 if (N.getOpcode() == ISD::SUB) in SelectT2AddrModeImm7()
1426 CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32); in SelectT2AddrModeImm7()
1432 Base = N; in SelectT2AddrModeImm7()
1433 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectT2AddrModeImm7()
1438 bool ARMDAGToDAGISel::SelectT2AddrModeImm7Offset(SDNode *Op, SDValue N, in SelectT2AddrModeImm7Offset() argument
1440 return SelectT2AddrModeImm7Offset(Op, N, OffImm, Shift); in SelectT2AddrModeImm7Offset()
1443 bool ARMDAGToDAGISel::SelectT2AddrModeImm7Offset(SDNode *Op, SDValue N, in SelectT2AddrModeImm7Offset() argument
1467 if (isScaledConstantInRange(N, 1 << Shift, 0, 0x80, RHSC)) { in SelectT2AddrModeImm7Offset()
1470 ? CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32) in SelectT2AddrModeImm7Offset()
1471 : CurDAG->getTargetConstant(-RHSC * (1 << Shift), SDLoc(N), in SelectT2AddrModeImm7Offset()
1479 bool ARMDAGToDAGISel::SelectImmediateInRange(SDValue N, SDValue &OffImm) { in SelectImmediateInRange() argument
1481 if (isScaledConstantInRange(N, 1, Min, Max, Val)) { in SelectImmediateInRange()
1482 OffImm = CurDAG->getTargetConstant(Val, SDLoc(N), MVT::i32); in SelectImmediateInRange()
1488 bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N, in SelectT2AddrModeSoReg() argument
1492 if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) in SelectT2AddrModeSoReg()
1496 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { in SelectT2AddrModeSoReg()
1506 Base = N.getOperand(0); in SelectT2AddrModeSoReg()
1507 OffReg = N.getOperand(1); in SelectT2AddrModeSoReg()
1532 if (OffReg.getOpcode() == ISD::MUL && N.hasOneUse()) { in SelectT2AddrModeSoReg()
1543 ShImm = CurDAG->getTargetConstant(ShAmt, SDLoc(N), MVT::i32); in SelectT2AddrModeSoReg()
1548 bool ARMDAGToDAGISel::SelectT2AddrModeExclusive(SDValue N, SDValue &Base, in SelectT2AddrModeExclusive() argument
1552 Base = N; in SelectT2AddrModeExclusive()
1553 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectT2AddrModeExclusive()
1555 if (N.getOpcode() != ISD::ADD || !CurDAG->isBaseWithConstantOffset(N)) in SelectT2AddrModeExclusive()
1558 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); in SelectT2AddrModeExclusive()
1566 Base = N.getOperand(0); in SelectT2AddrModeExclusive()
1573 OffImm = CurDAG->getTargetConstant(RHSC/4, SDLoc(N), MVT::i32); in SelectT2AddrModeExclusive()
1584 void ARMDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) { in transferMemOperands() argument
1585 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand(); in transferMemOperands()
1589 bool ARMDAGToDAGISel::tryARMIndexedLoad(SDNode *N) { in tryARMIndexedLoad() argument
1590 LoadSDNode *LD = cast<LoadSDNode>(N); in tryARMIndexedLoad()
1601 SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) { in tryARMIndexedLoad()
1605 SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) { in tryARMIndexedLoad()
1609 SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) { in tryARMIndexedLoad()
1614 SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) { in tryARMIndexedLoad()
1621 if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) { in tryARMIndexedLoad()
1627 SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) { in tryARMIndexedLoad()
1631 SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) { in tryARMIndexedLoad()
1634 } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) { in tryARMIndexedLoad()
1645 SDValue Ops[]= { Base, AMOpc, getAL(CurDAG, SDLoc(N)), in tryARMIndexedLoad()
1647 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32, in tryARMIndexedLoad()
1649 transferMemOperands(N, New); in tryARMIndexedLoad()
1650 ReplaceNode(N, New); in tryARMIndexedLoad()
1655 SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG, SDLoc(N)), in tryARMIndexedLoad()
1657 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32, in tryARMIndexedLoad()
1659 transferMemOperands(N, New); in tryARMIndexedLoad()
1660 ReplaceNode(N, New); in tryARMIndexedLoad()
1668 bool ARMDAGToDAGISel::tryT1IndexedLoad(SDNode *N) { in tryT1IndexedLoad() argument
1669 LoadSDNode *LD = cast<LoadSDNode>(N); in tryT1IndexedLoad()
1686 SDValue Ops[]= { Base, getAL(CurDAG, SDLoc(N)), in tryT1IndexedLoad()
1688 SDNode *New = CurDAG->getMachineNode(ARM::tLDR_postidx, SDLoc(N), MVT::i32, in tryT1IndexedLoad()
1690 transferMemOperands(N, New); in tryT1IndexedLoad()
1691 ReplaceNode(N, New); in tryT1IndexedLoad()
1695 bool ARMDAGToDAGISel::tryT2IndexedLoad(SDNode *N) { in tryT2IndexedLoad() argument
1696 LoadSDNode *LD = cast<LoadSDNode>(N); in tryT2IndexedLoad()
1707 if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) { in tryT2IndexedLoad()
1734 SDValue Ops[]= { Base, Offset, getAL(CurDAG, SDLoc(N)), in tryT2IndexedLoad()
1736 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32, in tryT2IndexedLoad()
1738 transferMemOperands(N, New); in tryT2IndexedLoad()
1739 ReplaceNode(N, New); in tryT2IndexedLoad()
1746 bool ARMDAGToDAGISel::tryMVEIndexedLoad(SDNode *N) { in tryMVEIndexedLoad() argument
1755 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) { in tryMVEIndexedLoad()
1771 } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(N)) { in tryMVEIndexedLoad()
1793 bool CanChangeType = Subtarget->isLittle() && !isa<MaskedLoadSDNode>(N); in tryMVEIndexedLoad()
1797 SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 1)) { in tryMVEIndexedLoad()
1803 SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 0)) { in tryMVEIndexedLoad()
1809 SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 0)) { in tryMVEIndexedLoad()
1817 SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 2)) in tryMVEIndexedLoad()
1822 SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 1)) in tryMVEIndexedLoad()
1825 SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 0)) in tryMVEIndexedLoad()
1832 CurDAG->getTargetConstant(Pred, SDLoc(N), MVT::i32), in tryMVEIndexedLoad()
1836 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, in tryMVEIndexedLoad()
1837 N->getValueType(0), MVT::Other, Ops); in tryMVEIndexedLoad()
1838 transferMemOperands(N, New); in tryMVEIndexedLoad()
1839 ReplaceUses(SDValue(N, 0), SDValue(New, 1)); in tryMVEIndexedLoad()
1840 ReplaceUses(SDValue(N, 1), SDValue(New, 0)); in tryMVEIndexedLoad()
1841 ReplaceUses(SDValue(N, 2), SDValue(New, 2)); in tryMVEIndexedLoad()
1842 CurDAG->RemoveDeadNode(N); in tryMVEIndexedLoad()
2109 void ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs, in SelectVLD() argument
2115 SDLoc dl(N); in SelectVLD()
2121 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) in SelectVLD()
2124 SDValue Chain = N->getOperand(0); in SelectVLD()
2125 EVT VT = N->getValueType(0); in SelectVLD()
2178 SDValue Inc = N->getOperand(AddrOpIdx + 1); in SelectVLD()
2214 SDValue Inc = N->getOperand(AddrOpIdx + 1); in SelectVLD()
2228 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); in SelectVLD()
2232 ReplaceNode(N, VLd); in SelectVLD()
2243 ReplaceUses(SDValue(N, Vec), in SelectVLD()
2245 ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1)); in SelectVLD()
2247 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2)); in SelectVLD()
2248 CurDAG->RemoveDeadNode(N); in SelectVLD()
2251 void ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs, in SelectVST() argument
2257 SDLoc dl(N); in SelectVST()
2264 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) in SelectVST()
2267 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); in SelectVST()
2269 SDValue Chain = N->getOperand(0); in SelectVST()
2270 EVT VT = N->getOperand(Vec0Idx).getValueType(); in SelectVST()
2309 SrcReg = N->getOperand(Vec0Idx); in SelectVST()
2312 SDValue V0 = N->getOperand(Vec0Idx + 0); in SelectVST()
2313 SDValue V1 = N->getOperand(Vec0Idx + 1); in SelectVST()
2317 SDValue V2 = N->getOperand(Vec0Idx + 2); in SelectVST()
2322 : N->getOperand(Vec0Idx + 3); in SelectVST()
2327 SDValue Q0 = N->getOperand(Vec0Idx); in SelectVST()
2328 SDValue Q1 = N->getOperand(Vec0Idx + 1); in SelectVST()
2337 SDValue Inc = N->getOperand(AddrOpIdx + 1); in SelectVST()
2360 ReplaceNode(N, VSt); in SelectVST()
2368 SDValue V0 = N->getOperand(Vec0Idx + 0); in SelectVST()
2369 SDValue V1 = N->getOperand(Vec0Idx + 1); in SelectVST()
2370 SDValue V2 = N->getOperand(Vec0Idx + 2); in SelectVST()
2373 : N->getOperand(Vec0Idx + 3); in SelectVST()
2389 SDValue Inc = N->getOperand(AddrOpIdx + 1); in SelectVST()
2402 ReplaceNode(N, VStB); in SelectVST()
2405 void ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating, in SelectVLDSTLane() argument
2411 SDLoc dl(N); in SelectVLDSTLane()
2418 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) in SelectVLDSTLane()
2421 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); in SelectVLDSTLane()
2423 SDValue Chain = N->getOperand(0); in SelectVLDSTLane()
2424 unsigned Lane = N->getConstantOperandVal(Vec0Idx + NumVecs); in SelectVLDSTLane()
2425 EVT VT = N->getOperand(Vec0Idx).getValueType(); in SelectVLDSTLane()
2480 SDValue Inc = N->getOperand(AddrOpIdx + 1); in SelectVLDSTLane()
2487 SDValue V0 = N->getOperand(Vec0Idx + 0); in SelectVLDSTLane()
2488 SDValue V1 = N->getOperand(Vec0Idx + 1); in SelectVLDSTLane()
2495 SDValue V2 = N->getOperand(Vec0Idx + 2); in SelectVLDSTLane()
2498 : N->getOperand(Vec0Idx + 3); in SelectVLDSTLane()
2515 ReplaceNode(N, VLdLn); in SelectVLDSTLane()
2526 ReplaceUses(SDValue(N, Vec), in SelectVLDSTLane()
2528 ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1)); in SelectVLDSTLane()
2530 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2)); in SelectVLDSTLane()
2531 CurDAG->RemoveDeadNode(N); in SelectVLDSTLane()
2569 void ARMDAGToDAGISel::SelectMVE_WB(SDNode *N, const uint16_t *Opcodes, in SelectMVE_WB() argument
2571 SDLoc Loc(N); in SelectMVE_WB()
2575 switch (N->getValueType(1).getVectorElementType().getSizeInBits()) { in SelectMVE_WB()
2586 Ops.push_back(N->getOperand(2)); // vector of base addresses in SelectMVE_WB()
2588 int32_t ImmValue = N->getConstantOperandVal(3); in SelectMVE_WB()
2592 AddMVEPredicateToOps(Ops, Loc, N->getOperand(4)); in SelectMVE_WB()
2596 Ops.push_back(N->getOperand(0)); // chain in SelectMVE_WB()
2599 VTs.push_back(N->getValueType(1)); in SelectMVE_WB()
2600 VTs.push_back(N->getValueType(0)); in SelectMVE_WB()
2601 VTs.push_back(N->getValueType(2)); in SelectMVE_WB()
2603 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), VTs, Ops); in SelectMVE_WB()
2604 ReplaceUses(SDValue(N, 0), SDValue(New, 1)); in SelectMVE_WB()
2605 ReplaceUses(SDValue(N, 1), SDValue(New, 0)); in SelectMVE_WB()
2606 ReplaceUses(SDValue(N, 2), SDValue(New, 2)); in SelectMVE_WB()
2607 transferMemOperands(N, New); in SelectMVE_WB()
2608 CurDAG->RemoveDeadNode(N); in SelectMVE_WB()
2611 void ARMDAGToDAGISel::SelectMVE_LongShift(SDNode *N, uint16_t Opcode, in SelectMVE_LongShift() argument
2614 SDLoc Loc(N); in SelectMVE_LongShift()
2618 Ops.push_back(N->getOperand(1)); in SelectMVE_LongShift()
2619 Ops.push_back(N->getOperand(2)); in SelectMVE_LongShift()
2623 int32_t ImmValue = N->getConstantOperandVal(3); in SelectMVE_LongShift()
2626 Ops.push_back(N->getOperand(3)); in SelectMVE_LongShift()
2631 int32_t SatOp = N->getConstantOperandVal(4); in SelectMVE_LongShift()
2641 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), ArrayRef(Ops)); in SelectMVE_LongShift()
2644 void ARMDAGToDAGISel::SelectMVE_VADCSBC(SDNode *N, uint16_t OpcodeWithCarry, in SelectMVE_VADCSBC() argument
2647 SDLoc Loc(N); in SelectMVE_VADCSBC()
2654 Ops.push_back(N->getOperand(FirstInputOp)); in SelectMVE_VADCSBC()
2655 Ops.push_back(N->getOperand(FirstInputOp + 1)); in SelectMVE_VADCSBC()
2656 SDValue CarryIn = N->getOperand(FirstInputOp + 2); in SelectMVE_VADCSBC()
2670 N->getOperand(FirstInputOp + 3), // predicate in SelectMVE_VADCSBC()
2671 N->getOperand(FirstInputOp - 1)); // inactive in SelectMVE_VADCSBC()
2673 AddEmptyMVEPredicateToOps(Ops, Loc, N->getValueType(0)); in SelectMVE_VADCSBC()
2675 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), ArrayRef(Ops)); in SelectMVE_VADCSBC()
2678 void ARMDAGToDAGISel::SelectMVE_VSHLC(SDNode *N, bool Predicated) { in SelectMVE_VSHLC() argument
2679 SDLoc Loc(N); in SelectMVE_VSHLC()
2684 Ops.push_back(N->getOperand(1)); in SelectMVE_VSHLC()
2685 Ops.push_back(N->getOperand(2)); in SelectMVE_VSHLC()
2686 int32_t ImmValue = N->getConstantOperandVal(3); in SelectMVE_VSHLC()
2690 AddMVEPredicateToOps(Ops, Loc, N->getOperand(4)); in SelectMVE_VSHLC()
2694 CurDAG->SelectNodeTo(N, ARM::MVE_VSHLC, N->getVTList(), ArrayRef(Ops)); in SelectMVE_VSHLC()
2705 void ARMDAGToDAGISel::SelectBaseMVE_VMLLDAV(SDNode *N, bool Predicated, in SelectBaseMVE_VMLLDAV() argument
2710 bool IsUnsigned = SDValueToConstBool(N->getOperand(1)); in SelectBaseMVE_VMLLDAV()
2711 bool IsSub = SDValueToConstBool(N->getOperand(2)); in SelectBaseMVE_VMLLDAV()
2712 bool IsExchange = SDValueToConstBool(N->getOperand(3)); in SelectBaseMVE_VMLLDAV()
2720 auto OpIsZero = [N](size_t OpNo) { in SelectBaseMVE_VMLLDAV()
2721 return isNullConstant(N->getOperand(OpNo)); in SelectBaseMVE_VMLLDAV()
2737 SDLoc Loc(N); in SelectBaseMVE_VMLLDAV()
2741 Ops.push_back(N->getOperand(4)); in SelectBaseMVE_VMLLDAV()
2742 Ops.push_back(N->getOperand(5)); in SelectBaseMVE_VMLLDAV()
2745 Ops.push_back(N->getOperand(6)); in SelectBaseMVE_VMLLDAV()
2746 Ops.push_back(N->getOperand(7)); in SelectBaseMVE_VMLLDAV()
2749 AddMVEPredicateToOps(Ops, Loc, N->getOperand(8)); in SelectBaseMVE_VMLLDAV()
2753 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), ArrayRef(Ops)); in SelectBaseMVE_VMLLDAV()
2756 void ARMDAGToDAGISel::SelectMVE_VMLLDAV(SDNode *N, bool Predicated, in SelectMVE_VMLLDAV() argument
2759 EVT VecTy = N->getOperand(6).getValueType(); in SelectMVE_VMLLDAV()
2772 SelectBaseMVE_VMLLDAV(N, Predicated, OpcodesS, OpcodesU, 2, SizeIndex); in SelectMVE_VMLLDAV()
2775 void ARMDAGToDAGISel::SelectMVE_VRMLLDAVH(SDNode *N, bool Predicated, in SelectMVE_VRMLLDAVH() argument
2779 N->getOperand(6).getValueType().getVectorElementType().getSizeInBits() == in SelectMVE_VRMLLDAVH()
2782 SelectBaseMVE_VMLLDAV(N, Predicated, OpcodesS, OpcodesU, 1, 0); in SelectMVE_VRMLLDAVH()
2785 void ARMDAGToDAGISel::SelectMVE_VLD(SDNode *N, unsigned NumVecs, in SelectMVE_VLD() argument
2788 EVT VT = N->getValueType(0); in SelectMVE_VLD()
2789 SDLoc Loc(N); in SelectMVE_VLD()
2812 SDValue Chain = N->getOperand(0); in SelectMVE_VLD()
2815 SDValue Ops[] = {Data, N->getOperand(PtrOperand), Chain}; in SelectMVE_VLD()
2820 transferMemOperands(N, LoadInst); in SelectMVE_VLD()
2825 SDValue Ops[] = {Data, N->getOperand(PtrOperand), Chain}; in SelectMVE_VLD()
2828 transferMemOperands(N, LoadInst); in SelectMVE_VLD()
2832 ReplaceUses(SDValue(N, i), in SelectMVE_VLD()
2836 ReplaceUses(SDValue(N, i++), SDValue(LoadInst, 1)); in SelectMVE_VLD()
2837 ReplaceUses(SDValue(N, i), SDValue(LoadInst, HasWriteback ? 2 : 1)); in SelectMVE_VLD()
2838 CurDAG->RemoveDeadNode(N); in SelectMVE_VLD()
2841 void ARMDAGToDAGISel::SelectMVE_VxDUP(SDNode *N, const uint16_t *Opcodes, in SelectMVE_VxDUP() argument
2843 EVT VT = N->getValueType(0); in SelectMVE_VxDUP()
2844 SDLoc Loc(N); in SelectMVE_VxDUP()
2866 Inactive = N->getOperand(OpIdx++); in SelectMVE_VxDUP()
2868 Ops.push_back(N->getOperand(OpIdx++)); // base in SelectMVE_VxDUP()
2870 Ops.push_back(N->getOperand(OpIdx++)); // limit in SelectMVE_VxDUP()
2872 SDValue ImmOp = N->getOperand(OpIdx++); // step in SelectMVE_VxDUP()
2877 AddMVEPredicateToOps(Ops, Loc, N->getOperand(OpIdx), Inactive); in SelectMVE_VxDUP()
2879 AddEmptyMVEPredicateToOps(Ops, Loc, N->getValueType(0)); in SelectMVE_VxDUP()
2881 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), ArrayRef(Ops)); in SelectMVE_VxDUP()
2884 void ARMDAGToDAGISel::SelectCDE_CXxD(SDNode *N, uint16_t Opcode, in SelectCDE_CXxD() argument
2887 SDLoc Loc(N); in SelectCDE_CXxD()
2893 SDValue ImmCorpoc = N->getOperand(OpIdx++); in SelectCDE_CXxD()
2900 SDValue AccLo = N->getOperand(OpIdx++); in SelectCDE_CXxD()
2901 SDValue AccHi = N->getOperand(OpIdx++); in SelectCDE_CXxD()
2909 Ops.push_back(N->getOperand(OpIdx++)); in SelectCDE_CXxD()
2912 SDValue Imm = N->getOperand(OpIdx); in SelectCDE_CXxD()
2937 if (SDValue(N, ResIdx).use_empty()) in SelectCDE_CXxD()
2941 ReplaceUses(SDValue(N, ResIdx), SubReg); in SelectCDE_CXxD()
2944 CurDAG->RemoveDeadNode(N); in SelectCDE_CXxD()
2947 void ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool IsIntrinsic, in SelectVLDDup() argument
2954 SDLoc dl(N); in SelectVLDDup()
2958 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) in SelectVLDDup()
2961 SDValue Chain = N->getOperand(0); in SelectVLDDup()
2962 EVT VT = N->getValueType(0); in SelectVLDDup()
3021 SDValue Inc = N->getOperand(2); in SelectVLDDup()
3057 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); in SelectVLDDup()
3062 ReplaceUses(SDValue(N, 0), SDValue(VLdDup, 0)); in SelectVLDDup()
3068 ReplaceUses(SDValue(N, Vec), in SelectVLDDup()
3072 ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1)); in SelectVLDDup()
3074 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2)); in SelectVLDDup()
3075 CurDAG->RemoveDeadNode(N); in SelectVLDDup()
3078 bool ARMDAGToDAGISel::tryInsertVectorElt(SDNode *N) { in tryInsertVectorElt() argument
3082 SDLoc dl(N); in tryInsertVectorElt()
3087 SDValue Ins1 = SDValue(N, 0); in tryInsertVectorElt()
3088 SDValue Ins2 = N->getOperand(0); in tryInsertVectorElt()
3169 bool ARMDAGToDAGISel::transformFixedFloatingPointConversion(SDNode *N, in transformFixedFloatingPointConversion() argument
3173 auto Type = N->getValueType(0); in transformFixedFloatingPointConversion()
3245 VecVal, CurDAG->getConstant(FracBits, SDLoc(N), MVT::i32)}; in transformFixedFloatingPointConversion()
3246 AddEmptyMVEPredicateToOps(Ops, SDLoc(N), Type); in transformFixedFloatingPointConversion()
3267 ReplaceNode(N, CurDAG->getMachineNode(Opcode, SDLoc(N), Type, Ops)); in transformFixedFloatingPointConversion()
3271 bool ARMDAGToDAGISel::tryFP_TO_INT(SDNode *N, SDLoc dl) { in tryFP_TO_INT() argument
3275 EVT Type = N->getValueType(0); in tryFP_TO_INT()
3280 bool IsUnsigned = N->getOpcode() == ISD::FP_TO_UINT || in tryFP_TO_INT()
3281 N->getOpcode() == ISD::FP_TO_UINT_SAT; in tryFP_TO_INT()
3282 SDNode *Node = N->getOperand(0).getNode(); in tryFP_TO_INT()
3308 ReplaceNode(N, CurDAG->getMachineNode(Opcode, dl, Type, Ops)); in tryFP_TO_INT()
3315 return transformFixedFloatingPointConversion(N, Node, IsUnsigned, false); in tryFP_TO_INT()
3318 bool ARMDAGToDAGISel::tryFMULFixed(SDNode *N, SDLoc dl) { in tryFMULFixed() argument
3322 auto Type = N->getValueType(0); in tryFMULFixed()
3326 auto LHS = N->getOperand(0); in tryFMULFixed()
3331 N, N, LHS.getOpcode() == ISD::UINT_TO_FP, true); in tryFMULFixed()
3334 bool ARMDAGToDAGISel::tryV6T2BitfieldExtractOp(SDNode *N, bool isSigned) { in tryV6T2BitfieldExtractOp() argument
3341 SDLoc dl(N); in tryV6T2BitfieldExtractOp()
3345 if (N->getOpcode() == ISD::AND) { in tryV6T2BitfieldExtractOp()
3346 if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) { in tryV6T2BitfieldExtractOp()
3353 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, in tryV6T2BitfieldExtractOp()
3368 if ((LSB + Width + 1) == N->getValueType(0).getSizeInBits()) { in tryV6T2BitfieldExtractOp()
3372 SDValue Ops[] = { N->getOperand(0).getOperand(0), in tryV6T2BitfieldExtractOp()
3375 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); in tryV6T2BitfieldExtractOp()
3384 SDValue Ops[] = { N->getOperand(0).getOperand(0), ShOpc, in tryV6T2BitfieldExtractOp()
3386 CurDAG->SelectNodeTo(N, ARM::MOVsi, MVT::i32, Ops); in tryV6T2BitfieldExtractOp()
3391 SDValue Ops[] = { N->getOperand(0).getOperand(0), in tryV6T2BitfieldExtractOp()
3395 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); in tryV6T2BitfieldExtractOp()
3404 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) { in tryV6T2BitfieldExtractOp()
3407 if (isInt32Immediate(N->getOperand(1), Srl_imm)) { in tryV6T2BitfieldExtractOp()
3416 SDValue Ops[] = { N->getOperand(0).getOperand(0), in tryV6T2BitfieldExtractOp()
3420 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); in tryV6T2BitfieldExtractOp()
3426 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, And_imm) && in tryV6T2BitfieldExtractOp()
3431 if (isInt32Immediate(N->getOperand(1), Srl_imm) && Srl_imm == LSB) { in tryV6T2BitfieldExtractOp()
3438 SDValue Ops[] = { N->getOperand(0).getOperand(0), in tryV6T2BitfieldExtractOp()
3442 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); in tryV6T2BitfieldExtractOp()
3447 if (N->getOpcode() == ISD::SIGN_EXTEND_INREG) { in tryV6T2BitfieldExtractOp()
3448 unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits(); in tryV6T2BitfieldExtractOp()
3450 if (!isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, LSB) && in tryV6T2BitfieldExtractOp()
3451 !isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRA, LSB)) in tryV6T2BitfieldExtractOp()
3459 SDValue Ops[] = { N->getOperand(0).getOperand(0), in tryV6T2BitfieldExtractOp()
3463 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); in tryV6T2BitfieldExtractOp()
3480 bool ARMDAGToDAGISel::tryABSOp(SDNode *N){ in tryABSOp() argument
3481 SDValue SUBSrc0 = N->getOperand(0); in tryABSOp()
3482 SDValue SUBSrc1 = N->getOperand(1); in tryABSOp()
3483 EVT VT = N->getValueType(0); in tryABSOp()
3502 CurDAG->SelectNodeTo(N, Opcode, VT, XORSrc0); in tryABSOp()
3510 void ARMDAGToDAGISel::SelectCMP_SWAP(SDNode *N) { in SelectCMP_SWAP() argument
3512 EVT MemTy = cast<MemSDNode>(N)->getMemoryVT(); in SelectCMP_SWAP()
3522 SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3), in SelectCMP_SWAP()
3523 N->getOperand(0)}; in SelectCMP_SWAP()
3525 Opcode, SDLoc(N), in SelectCMP_SWAP()
3528 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand(); in SelectCMP_SWAP()
3531 ReplaceUses(SDValue(N, 0), SDValue(CmpSwap, 0)); in SelectCMP_SWAP()
3532 ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 2)); in SelectCMP_SWAP()
3533 CurDAG->RemoveDeadNode(N); in SelectCMP_SWAP()
3545 void ARMDAGToDAGISel::SelectCMPZ(SDNode *N, bool &SwitchEQNEToPLMI) { in SelectCMPZ() argument
3546 assert(N->getOpcode() == ARMISD::CMPZ); in SelectCMPZ()
3555 SDValue And = N->getOperand(0); in SelectCMPZ()
3559 SDValue Zero = N->getOperand(1); in SelectCMPZ()
3573 SDLoc dl(N); in SelectCMPZ()
3631 void ARMDAGToDAGISel::Select(SDNode *N) { in Select() argument
3632 SDLoc dl(N); in Select()
3634 if (N->isMachineOpcode()) { in Select()
3635 N->setNodeId(-1); in Select()
3639 switch (N->getOpcode()) { in Select()
3652 StoreSDNode *ST = cast<StoreSDNode>(N); in Select()
3673 ReplaceNode(N, ResNode); in Select()
3680 if (tryWriteRegister(N)) in Select()
3684 if (tryReadRegister(N)) in Select()
3689 if (tryInlineAsm(N)) in Select()
3694 if (tryABSOp(N)) in Select()
3699 unsigned Val = N->getAsZExtVal(); in Select()
3738 ReplaceNode(N, ResNode); in Select()
3747 int FI = cast<FrameIndexSDNode>(N)->getIndex(); in Select()
3756 CurDAG->SelectNodeTo(N, ARM::tADDframe, MVT::i32, TFI, in Select()
3765 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); in Select()
3770 if (tryInsertVectorElt(N)) in Select()
3775 if (tryV6T2BitfieldExtractOp(N, false)) in Select()
3780 if (tryV6T2BitfieldExtractOp(N, true)) in Select()
3787 if (tryFP_TO_INT(N, dl)) in Select()
3791 if (tryFMULFixed(N, dl)) in Select()
3797 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) { in Select()
3804 SDValue V = N->getOperand(0); in Select()
3810 CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops); in Select()
3815 CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops); in Select()
3823 SDValue V = N->getOperand(0); in Select()
3829 CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops); in Select()
3834 CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops); in Select()
3842 if (tryV6T2BitfieldExtractOp(N, false)) in Select()
3848 auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1)); in Select()
3868 CurDAG->RepositionNode(N->getIterator(), NewImm.getNode()); in Select()
3872 N->getOperand(0), NewImm, getAL(CurDAG, dl), in Select()
3874 ReplaceNode(N, CurDAG->getMachineNode(ARM::tBIC, dl, MVT::i32, Ops)); in Select()
3877 SDValue Ops[] = {N->getOperand(0), NewImm, getAL(CurDAG, dl), in Select()
3880 ReplaceNode(N, in Select()
3892 EVT VT = N->getValueType(0); in Select()
3900 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1); in Select()
3918 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, Ops)); in Select()
3927 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), in Select()
3928 N->getOperand(2), N->getOperand(3), in Select()
3931 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::i32, Ops)); in Select()
3936 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), in Select()
3937 N->getOperand(3), getAL(CurDAG, dl), in Select()
3940 N, CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops)); in Select()
3943 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), in Select()
3944 N->getOperand(3), getAL(CurDAG, dl), in Select()
3947 ReplaceNode(N, CurDAG->getMachineNode( in Select()
3955 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), in Select()
3956 N->getOperand(3), getAL(CurDAG, dl), in Select()
3959 N, CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops)); in Select()
3962 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), in Select()
3963 N->getOperand(3), getAL(CurDAG, dl), in Select()
3966 ReplaceNode(N, CurDAG->getMachineNode( in Select()
3977 if (N->getOperand(1).getOpcode() != ISD::SMUL_LOHI || in Select()
3978 N->getOperand(2).getOpcode() != ARMISD::SUBC || in Select()
3979 !SDValue(N, 1).use_empty()) in Select()
3986 SDValue SmulLoHi = N->getOperand(1); in Select()
3987 SDValue Subc = N->getOperand(2); in Select()
3991 N->getOperand(1) != SmulLoHi.getValue(1) || in Select()
3992 N->getOperand(2) != Subc.getValue(1)) in Select()
3997 N->getOperand(0), getAL(CurDAG, dl), in Select()
3999 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops)); in Select()
4003 if (Subtarget->hasMVEIntegerOps() && tryMVEIndexedLoad(N)) in Select()
4006 if (tryT2IndexedLoad(N)) in Select()
4009 if (tryT1IndexedLoad(N)) in Select()
4011 } else if (tryARMIndexedLoad(N)) in Select()
4017 if (Subtarget->hasMVEIntegerOps() && tryMVEIndexedLoad(N)) in Select()
4023 N->getOperand(0)); in Select()
4024 ReplaceUses(N, New); in Select()
4025 CurDAG->RemoveDeadNode(N); in Select()
4030 N->getOperand(1), N->getOperand(2), in Select()
4031 N->getOperand(0)); in Select()
4032 ReplaceUses(N, New); in Select()
4033 CurDAG->RemoveDeadNode(N); in Select()
4037 SDValue Ops[] = { N->getOperand(1), in Select()
4038 N->getOperand(2), in Select()
4039 N->getOperand(0) }; in Select()
4042 ReplaceUses(N, New); in Select()
4043 CurDAG->RemoveDeadNode(N); in Select()
4050 const SDValue &Chain = N->getOperand(0); in Select()
4051 const SDValue &Addr = N->getOperand(1); in Select()
4068 transferMemOperands(N, New); in Select()
4069 ReplaceUses(SDValue(N, 0), Lo); in Select()
4070 ReplaceUses(SDValue(N, 1), Hi); in Select()
4071 ReplaceUses(SDValue(N, 2), SDValue(New, 1)); in Select()
4072 CurDAG->RemoveDeadNode(N); in Select()
4079 const SDValue &Chain = N->getOperand(0); in Select()
4080 const SDValue &Addr = N->getOperand(3); in Select()
4091 createGPRPairNode(MVT::Untyped, N->getOperand(1), N->getOperand(2)); in Select()
4094 transferMemOperands(N, New); in Select()
4095 ReplaceUses(SDValue(N, 0), SDValue(New, 0)); in Select()
4096 CurDAG->RemoveDeadNode(N); in Select()
4100 SDValue Ops[] = { N->getOperand(1), in Select()
4101 N->getOperand(2), in Select()
4102 N->getOperand(0) }; in Select()
4106 ReplaceUses(N, Dec); in Select()
4107 CurDAG->RemoveDeadNode(N); in Select()
4125 SDValue Chain = N->getOperand(0); in Select()
4126 SDValue N1 = N->getOperand(1); in Select()
4127 SDValue N2 = N->getOperand(2); in Select()
4128 SDValue N3 = N->getOperand(3); in Select()
4129 SDValue InGlue = N->getOperand(4); in Select()
4158 ReplaceUses(N, LoopEnd); in Select()
4159 CurDAG->RemoveDeadNode(N); in Select()
4168 InGlue = N->getOperand(4); in Select()
4188 if (N->getNumValues() == 2) { in Select()
4190 ReplaceUses(SDValue(N, 1), InGlue); in Select()
4192 ReplaceUses(SDValue(N, 0), in Select()
4194 CurDAG->RemoveDeadNode(N); in Select()
4203 SDValue X = N->getOperand(0); in Select()
4204 auto *C = dyn_cast<ConstantSDNode>(N->getOperand(1).getNode()); in Select()
4229 CurDAG->MorphNodeTo(N, ARMISD::CMPZ, CurDAG->getVTList(MVT::Glue), Ops2); in Select()
4237 SDValue InGlue = N->getOperand(4); in Select()
4244 SDValue ARMcc = N->getOperand(2); in Select()
4257 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), NewARMcc, in Select()
4258 N->getOperand(3), N->getOperand(4)}; in Select()
4259 CurDAG->MorphNodeTo(N, ARMISD::CMOV, N->getVTList(), Ops); in Select()
4267 EVT VT = N->getValueType(0); in Select()
4274 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), Pred, PredReg}; in Select()
4275 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops)); in Select()
4279 EVT VT = N->getValueType(0); in Select()
4286 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), Pred, PredReg}; in Select()
4287 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops)); in Select()
4291 EVT VT = N->getValueType(0); in Select()
4297 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), Pred, PredReg}; in Select()
4298 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops)); in Select()
4302 EVT VecVT = N->getValueType(0); in Select()
4308 N, createDRegPairNode(VecVT, N->getOperand(0), N->getOperand(1))); in Select()
4314 N, createSRegPairNode(VecVT, N->getOperand(0), N->getOperand(1))); in Select()
4318 ReplaceNode(N, in Select()
4319 createQuadSRegsNode(VecVT, N->getOperand(0), N->getOperand(1), in Select()
4320 N->getOperand(2), N->getOperand(3))); in Select()
4329 SelectVLDDup(N, /* IsIntrinsic= */ false, false, 1, DOpcodes, QOpcodes); in Select()
4336 SelectVLDDup(N, /* IsIntrinsic= */ false, false, 2, Opcodes); in Select()
4344 SelectVLDDup(N, /* IsIntrinsic= */ false, false, 3, Opcodes); in Select()
4352 SelectVLDDup(N, /* IsIntrinsic= */ false, false, 4, Opcodes); in Select()
4363 SelectVLDDup(N, /* IsIntrinsic= */ false, true, 1, DOpcodes, QOpcodes); in Select()
4378 SelectVLDDup(N, /* IsIntrinsic= */ false, true, 2, DOpcodes, QOpcodes0, QOpcodes1); in Select()
4393 SelectVLDDup(N, /* IsIntrinsic= */ false, true, 3, DOpcodes, QOpcodes0, QOpcodes1); in Select()
4408 SelectVLDDup(N, /* IsIntrinsic= */ false, true, 4, DOpcodes, QOpcodes0, QOpcodes1); in Select()
4421 SelectVLD(N, true, 1, DOpcodes, QOpcodes, nullptr); in Select()
4433 SelectVLD(N, true, 2, DOpcodes, QOpcodes, nullptr); in Select()
4442 SelectMVE_VLD(N, 2, Opcodes, true); in Select()
4458 SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1); in Select()
4473 SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1); in Select()
4485 SelectMVE_VLD(N, 4, Opcodes, true); in Select()
4498 SelectVLD(N, true, 2, DOpcodes, QOpcodes, nullptr); in Select()
4515 SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1); in Select()
4532 SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1); in Select()
4544 SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes); in Select()
4554 SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes); in Select()
4564 SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes); in Select()
4577 SelectVST(N, true, 1, DOpcodes, QOpcodes, nullptr); in Select()
4589 SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr); in Select()
4606 SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1); in Select()
4621 SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1); in Select()
4637 SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr); in Select()
4657 SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1); in Select()
4677 SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1); in Select()
4688 SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes); in Select()
4698 SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes); in Select()
4708 SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes); in Select()
4714 unsigned IntNo = N->getConstantOperandVal(1); in Select()
4721 SDLoc dl(N); in Select()
4722 SDValue Chain = N->getOperand(0); in Select()
4731 Ops.push_back(getI32Imm(N->getConstantOperandVal(2), dl)); /* coproc */ in Select()
4732 Ops.push_back(getI32Imm(N->getConstantOperandVal(3), dl)); /* opc */ in Select()
4733 Ops.push_back(getI32Imm(N->getConstantOperandVal(4), dl)); /* CRm */ in Select()
4748 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, RetType, Ops)); in Select()
4753 SDLoc dl(N); in Select()
4754 SDValue Chain = N->getOperand(0); in Select()
4755 SDValue MemAddr = N->getOperand(2); in Select()
4776 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); in Select()
4781 if (!SDValue(N, 0).use_empty()) { in Select()
4792 ReplaceUses(SDValue(N, 0), Result); in Select()
4794 if (!SDValue(N, 1).use_empty()) { in Select()
4805 ReplaceUses(SDValue(N, 1), Result); in Select()
4807 ReplaceUses(SDValue(N, 2), OutChain); in Select()
4808 CurDAG->RemoveDeadNode(N); in Select()
4813 SDLoc dl(N); in Select()
4814 SDValue Chain = N->getOperand(0); in Select()
4815 SDValue Val0 = N->getOperand(2); in Select()
4816 SDValue Val1 = N->getOperand(3); in Select()
4817 SDValue MemAddr = N->getOperand(4); in Select()
4843 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); in Select()
4846 ReplaceNode(N, St); in Select()
4855 SelectVLD(N, false, 1, DOpcodes, QOpcodes, nullptr); in Select()
4866 SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr); in Select()
4883 SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1); in Select()
4900 SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1); in Select()
4909 SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr); in Select()
4924 SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1); in Select()
4939 SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1); in Select()
4952 SelectVLDDup(N, /* IsIntrinsic= */ true, false, 2, in Select()
4968 SelectVLDDup(N, /* IsIntrinsic= */ true, false, 3, in Select()
4984 SelectVLDDup(N, /* IsIntrinsic= */ true, false, 4, in Select()
4995 SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes); in Select()
5005 SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes); in Select()
5015 SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes); in Select()
5024 SelectVST(N, false, 1, DOpcodes, QOpcodes, nullptr); in Select()
5035 SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr); in Select()
5052 SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1); in Select()
5069 SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1); in Select()
5078 SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr); in Select()
5093 SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1); in Select()
5108 SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1); in Select()
5118 SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes); in Select()
5128 SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes); in Select()
5138 SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes); in Select()
5146 SelectMVE_WB(N, Opcodes, in Select()
5158 SelectMVE_VLD(N, 2, Opcodes, false); in Select()
5172 SelectMVE_VLD(N, 4, Opcodes, false); in Select()
5180 unsigned IntNo = N->getConstantOperandVal(0); in Select()
5187 SDLoc dl(N); in Select()
5188 const SDValue &Src = N->getOperand(1); in Select()
5189 llvm::EVT DestTy = N->getValueType(0); in Select()
5193 CurDAG->SelectNodeTo(N, ARM::BF16_VCVTB, DestTy, Ops); in Select()
5199 SDLoc dl(N); in Select()
5200 const SDValue &Src = N->getOperand(1); in Select()
5204 CurDAG->SelectNodeTo(N, ARM::BF16_VCVT, MVT::v4bf16, Ops); in Select()
5209 SelectMVE_LongShift(N, ARM::MVE_URSHRL, true, false); in Select()
5212 SelectMVE_LongShift(N, ARM::MVE_UQSHLL, true, false); in Select()
5215 SelectMVE_LongShift(N, ARM::MVE_SRSHRL, true, false); in Select()
5218 SelectMVE_LongShift(N, ARM::MVE_SQSHLL, true, false); in Select()
5221 SelectMVE_LongShift(N, ARM::MVE_UQRSHLL, false, true); in Select()
5224 SelectMVE_LongShift(N, ARM::MVE_SQRSHRL, false, true); in Select()
5229 SelectMVE_VADCSBC(N, ARM::MVE_VADC, ARM::MVE_VADCI, true, in Select()
5234 SelectMVE_VADCSBC(N, ARM::MVE_VSBC, ARM::MVE_VSBCI, true, in Select()
5239 SelectMVE_VSHLC(N, IntNo == Intrinsic::arm_mve_vshlc_predicated); in Select()
5258 SelectMVE_VMLLDAV(N, IntNo == Intrinsic::arm_mve_vmlldava_predicated, in Select()
5274 SelectMVE_VRMLLDAVH(N, IntNo == Intrinsic::arm_mve_vrmlldavha_predicated, in Select()
5284 SelectMVE_VxDUP(N, Opcodes, false, in Select()
5294 SelectMVE_VxDUP(N, Opcodes, false, in Select()
5304 SelectMVE_VxDUP(N, Opcodes, true, in Select()
5314 SelectMVE_VxDUP(N, Opcodes, true, in Select()
5349 SelectCDE_CXxD(N, Opcode, NumExtraOps, HasAccum); in Select()
5357 SelectCMP_SWAP(N); in Select()
5361 SelectCode(N); in Select()
5489 bool ARMDAGToDAGISel::tryReadRegister(SDNode *N){ in tryReadRegister() argument
5490 const auto *MD = cast<MDNodeSDNode>(N->getOperand(1)); in tryReadRegister()
5493 SDLoc DL(N); in tryReadRegister()
5517 Ops.push_back(N->getOperand(0)); in tryReadRegister()
5518 ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, ResTypes, Ops)); in tryReadRegister()
5528 N->getOperand(0) }; in tryReadRegister()
5530 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSbanked : ARM::MRSbanked, in tryReadRegister()
5557 N->getOperand(0) }; in tryReadRegister()
5558 ReplaceNode(N, in tryReadRegister()
5573 N->getOperand(0) }; in tryReadRegister()
5575 N, CurDAG->getMachineNode(ARM::t2MRS_M, DL, MVT::i32, MVT::Other, Ops)); in tryReadRegister()
5583 N->getOperand(0) }; in tryReadRegister()
5584 ReplaceNode(N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRS_AR : ARM::MRS, in tryReadRegister()
5591 N->getOperand(0) }; in tryReadRegister()
5593 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSsys_AR : ARM::MRSsys, DL, in tryReadRegister()
5604 bool ARMDAGToDAGISel::tryWriteRegister(SDNode *N){ in tryWriteRegister() argument
5605 const auto *MD = cast<MDNodeSDNode>(N->getOperand(1)); in tryWriteRegister()
5608 SDLoc DL(N); in tryWriteRegister()
5621 Ops.insert(Ops.begin()+2, N->getOperand(2)); in tryWriteRegister()
5626 SDValue WriteValue[] = { N->getOperand(2), N->getOperand(3) }; in tryWriteRegister()
5632 Ops.push_back(N->getOperand(0)); in tryWriteRegister()
5634 ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops)); in tryWriteRegister()
5641 Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32), N->getOperand(2), in tryWriteRegister()
5643 N->getOperand(0) }; in tryWriteRegister()
5645 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSRbanked : ARM::MSRbanked, in tryWriteRegister()
5664 Ops = { N->getOperand(2), getAL(CurDAG, DL), in tryWriteRegister()
5665 CurDAG->getRegister(0, MVT::i32), N->getOperand(0) }; in tryWriteRegister()
5666 ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops)); in tryWriteRegister()
5683 N->getOperand(2), getAL(CurDAG, DL), in tryWriteRegister()
5684 CurDAG->getRegister(0, MVT::i32), N->getOperand(0) }; in tryWriteRegister()
5685 ReplaceNode(N, CurDAG->getMachineNode(ARM::t2MSR_M, DL, MVT::Other, Ops)); in tryWriteRegister()
5694 Ops = { CurDAG->getTargetConstant(Mask, DL, MVT::i32), N->getOperand(2), in tryWriteRegister()
5696 N->getOperand(0) }; in tryWriteRegister()
5697 ReplaceNode(N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSR_AR : ARM::MSR, in tryWriteRegister()
5705 bool ARMDAGToDAGISel::tryInlineAsm(SDNode *N){ in tryInlineAsm() argument
5709 unsigned NumOps = N->getNumOperands(); in tryInlineAsm()
5719 SDLoc dl(N); in tryInlineAsm()
5720 SDValue Glue = N->getGluedNode() ? N->getOperand(NumOps - 1) : SDValue(); in tryInlineAsm()
5724 for(unsigned i = 0, e = N->getGluedNode() ? NumOps - 1 : NumOps; i < e; ++i) { in tryInlineAsm()
5725 SDValue op = N->getOperand(i); in tryInlineAsm()
5731 if (const auto *C = dyn_cast<ConstantSDNode>(N->getOperand(i))) in tryInlineAsm()
5741 SDValue op = N->getOperand(++i); in tryInlineAsm()
5763 SDValue op = N->getOperand(++i); in tryInlineAsm()
5779 SDValue V0 = N->getOperand(i+1); in tryInlineAsm()
5780 SDValue V1 = N->getOperand(i+2); in tryInlineAsm()
5792 SDValue Chain = SDValue(N,0); in tryInlineAsm()
5794 SDNode *GU = N->getGluedUser(); in tryInlineAsm()
5857 SDValue New = CurDAG->getNode(N->getOpcode(), SDLoc(N), in tryInlineAsm()
5860 ReplaceNode(N, New.getNode()); in tryInlineAsm()