Lines Matching refs:DL

761 EVT SystemZTargetLowering::getSetCCResultType(const DataLayout &DL,  in getSetCCResultType()  argument
1059 bool SystemZTargetLowering::isLegalAddressingMode(const DataLayout &DL, in isLegalAddressingMode() argument
1453 static SDValue convertLocVTToValVT(SelectionDAG &DAG, const SDLoc &DL, in convertLocVTToValVT() argument
1459 Value = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Value, in convertLocVTToValVT()
1462 Value = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Value, in convertLocVTToValVT()
1466 Value = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Value); in convertLocVTToValVT()
1472 Value = DAG.getBuildVector(MVT::v2i64, DL, {Value, DAG.getUNDEF(MVT::i64)}); in convertLocVTToValVT()
1473 Value = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Value); in convertLocVTToValVT()
1482 static SDValue convertValVTToLocVT(SelectionDAG &DAG, const SDLoc &DL, in convertValVTToLocVT() argument
1486 return DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Value); in convertValVTToLocVT()
1488 return DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Value); in convertValVTToLocVT()
1490 return DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Value); in convertValVTToLocVT()
1498 Value = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f64, Value); in convertValVTToLocVT()
1502 Value = DAG.getNode(ISD::BITCAST, DL, BitCastToType, Value); in convertValVTToLocVT()
1506 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VA.getLocVT(), Value, in convertValVTToLocVT()
1507 DAG.getConstant(0, DL, MVT::i32)); in convertValVTToLocVT()
1518 SDLoc DL(In); in lowerI128ToGR128() local
1521 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, In); in lowerI128ToGR128()
1522 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, in lowerI128ToGR128()
1523 DAG.getNode(ISD::SRL, DL, MVT::i128, In, in lowerI128ToGR128()
1524 DAG.getConstant(64, DL, MVT::i32))); in lowerI128ToGR128()
1526 std::tie(Lo, Hi) = DAG.SplitScalar(In, DL, MVT::i64, MVT::i64); in lowerI128ToGR128()
1529 SDNode *Pair = DAG.getMachineNode(SystemZ::PAIR128, DL, in lowerI128ToGR128()
1535 SDLoc DL(In); in lowerGR128ToI128() local
1537 DL, MVT::i64, In); in lowerGR128ToI128()
1539 DL, MVT::i64, In); in lowerGR128ToI128()
1542 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i128, Lo); in lowerGR128ToI128()
1543 Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i128, Hi); in lowerGR128ToI128()
1544 Hi = DAG.getNode(ISD::SHL, DL, MVT::i128, Hi, in lowerGR128ToI128()
1545 DAG.getConstant(64, DL, MVT::i32)); in lowerGR128ToI128()
1546 return DAG.getNode(ISD::OR, DL, MVT::i128, Lo, Hi); in lowerGR128ToI128()
1548 return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i128, Lo, Hi); in lowerGR128ToI128()
1553 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, in splitValueIntoRegisterParts() argument
1566 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, in joinRegisterPartsIntoValue() argument
1579 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, in LowerFormalArguments() argument
1640 ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, LocVT); in LowerFormalArguments()
1661 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, in LowerFormalArguments()
1662 DAG.getIntPtrConstant(4, DL)); in LowerFormalArguments()
1663 ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN, in LowerFormalArguments()
1670 InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue, in LowerFormalArguments()
1679 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, in LowerFormalArguments()
1680 DAG.getIntPtrConstant(PartOffset, DL)); in LowerFormalArguments()
1681 InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address, in LowerFormalArguments()
1686 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue)); in LowerFormalArguments()
1735 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64); in LowerFormalArguments()
1736 MemOps[I] = DAG.getStore(ArgValue.getValue(1), DL, ArgValue, FIN, in LowerFormalArguments()
1740 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in LowerFormalArguments()
1779 static SDValue getADAEntry(SelectionDAG &DAG, SDValue Val, SDLoc DL, in getADAEntry() argument
1787 SDValue Ofs = DAG.getTargetConstant(Offset, DL, PtrVT); in getADAEntry()
1789 SDValue Result = DAG.getNode(SystemZISD::ADA_ENTRY, DL, PtrVT, Val, Reg, Ofs); in getADAEntry()
1792 PtrVT, DL, DAG.getEntryNode(), Result, MachinePointerInfo(), Align(8), in getADAEntry()
1800 static SDValue getADAEntry(SelectionDAG &DAG, const GlobalValue *GV, SDLoc DL, in getADAEntry() argument
1818 SDValue Val = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, ADAtype); in getADAEntry()
1820 return getADAEntry(DAG, Val, DL, 0, LoadAddr); in getADAEntry()
1824 SDLoc &DL, SDValue &Chain) { in getzOSCalleeAndADA() argument
1838 ADA = DAG.getCopyFromReg(Chain, DL, ADAvReg, PtrVT); in getzOSCalleeAndADA()
1839 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT); in getzOSCalleeAndADA()
1840 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); in getzOSCalleeAndADA()
1844 G->getGlobal(), DL, PtrVT, 0, SystemZII::MO_ADA_DIRECT_FUNC_DESC); in getzOSCalleeAndADA()
1845 ADA = getADAEntry(DAG, GA, DL, ADADelta); in getzOSCalleeAndADA()
1846 Callee = getADAEntry(DAG, GA, DL, EPADelta); in getzOSCalleeAndADA()
1851 ADA = getADAEntry(DAG, ES, DL, ADADelta); in getzOSCalleeAndADA()
1852 Callee = getADAEntry(DAG, ES, DL, EPADelta); in getzOSCalleeAndADA()
1855 ADA = DAG.getNode(ISD::ADD, DL, PtrVT, Callee, in getzOSCalleeAndADA()
1856 DAG.getConstant(ADADelta, DL, PtrVT)); in getzOSCalleeAndADA()
1857 ADA = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), ADA, in getzOSCalleeAndADA()
1859 Callee = DAG.getNode(ISD::ADD, DL, PtrVT, Callee, in getzOSCalleeAndADA()
1860 DAG.getConstant(EPADelta, DL, PtrVT)); in getzOSCalleeAndADA()
1861 Callee = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Callee, in getzOSCalleeAndADA()
1871 SDLoc &DL = CLI.DL; in LowerCall() local
1904 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL); in LowerCall()
1931 DAG.getStore(Chain, DL, ArgValue, SpillSlot, in LowerCall()
1939 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, in LowerCall()
1940 DAG.getIntPtrConstant(PartOffset, DL)); in LowerCall()
1942 DAG.getStore(Chain, DL, PartValue, Address, in LowerCall()
1950 ArgValue = convertValVTToLocVT(DAG, DL, VA, ArgValue); in LowerCall()
1966 StackPtr = DAG.getCopyFromReg(Chain, DL, in LowerCall()
1972 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, in LowerCall()
1973 DAG.getIntPtrConstant(Offset, DL)); in LowerCall()
1977 DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo())); in LowerCall()
1985 DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, ArgValue, in LowerCall()
1986 DAG.getIntPtrConstant(1, DL)); in LowerCall()
1994 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); in LowerCall()
2003 bool IsBRASL = getzOSCalleeAndADA(DAG, Callee, ADA, DL, Chain); in LowerCall()
2007 Chain = DAG.getCopyToReg(Chain, DL, CalleeReg, Callee, Glue); in LowerCall()
2015 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT); in LowerCall()
2016 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); in LowerCall()
2019 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); in LowerCall()
2021 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R1D, Callee, Glue); in LowerCall()
2029 Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first, in LowerCall()
2058 SDValue Ret = DAG.getNode(SystemZISD::SIBCALL, DL, NodeTys, Ops); in LowerCall()
2062 Chain = DAG.getNode(SystemZISD::CALL, DL, NodeTys, Ops); in LowerCall()
2067 Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, Glue, DL); in LowerCall()
2080 SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), in LowerCall()
2087 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, RetValue)); in LowerCall()
2097 ArrayRef<SDValue> Ops, CallingConv::ID CallConv, bool IsSigned, SDLoc DL, in makeExternalCall() argument
2117 CLI.setDebugLoc(DL) in makeExternalCall()
2148 const SDLoc &DL, SelectionDAG &DAG) const { in LowerReturn() argument
2158 return DAG.getNode(SystemZISD::RET_GLUE, DL, MVT::Other, Chain); in LowerReturn()
2175 RetValue = convertValVTToLocVT(DAG, DL, VA, RetValue); in LowerReturn()
2179 Chain = DAG.getCopyToReg(Chain, DL, Reg, RetValue, Glue); in LowerReturn()
2189 return DAG.getNode(SystemZISD::RET_GLUE, DL, MVT::Other, RetOps); in LowerReturn()
2443 static void adjustZeroCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) { in adjustZeroCmp() argument
2457 C.Op1 = DAG.getConstant(0, DL, C.Op1.getValueType()); in adjustZeroCmp()
2463 static void adjustSubwordCmp(SelectionDAG &DAG, const SDLoc &DL, in adjustSubwordCmp() argument
2534 C.Op1 = DAG.getConstant(Value, DL, MVT::i32); in adjustSubwordCmp()
2622 static void adjustForSubtraction(SelectionDAG &DAG, const SDLoc &DL, in adjustForSubtraction() argument
2637 C.Op1 = DAG.getConstant(0, DL, N->getValueType(0)); in adjustForSubtraction()
2693 static void adjustICmpTruncate(SelectionDAG &DAG, const SDLoc &DL, in adjustICmpTruncate() argument
2707 C.Op1 = DAG.getConstant(0, DL, C.Op0.getValueType()); in adjustICmpTruncate()
2825 static void adjustForTestUnderMask(SelectionDAG &DAG, const SDLoc &DL, in adjustForTestUnderMask() argument
2836 C.Op1 = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, C.Op0.getOperand(1)); in adjustForTestUnderMask()
2837 C.Op0 = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, C.Op0.getOperand(0)); in adjustForTestUnderMask()
2929 C.Op1 = DAG.getConstant(MaskVal, DL, C.Op0.getValueType()); in adjustForTestUnderMask()
2935 static void adjustICmp128(SelectionDAG &DAG, const SDLoc &DL, in adjustICmp128() argument
2946 C.Op0 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, C.Op0); in adjustICmp128()
2947 C.Op1 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, C.Op1); in adjustICmp128()
2982 static void adjustForRedundantAnd(SelectionDAG &DAG, const SDLoc &DL, in adjustForRedundantAnd() argument
3034 ISD::CondCode Cond, const SDLoc &DL, in getCmp() argument
3080 adjustForRedundantAnd(DAG, DL, C); in getCmp()
3081 adjustZeroCmp(DAG, DL, C); in getCmp()
3082 adjustSubwordCmp(DAG, DL, C); in getCmp()
3083 adjustForSubtraction(DAG, DL, C); in getCmp()
3085 adjustICmpTruncate(DAG, DL, C); in getCmp()
3093 adjustForTestUnderMask(DAG, DL, C); in getCmp()
3094 adjustICmp128(DAG, DL, C); in getCmp()
3099 static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) { in emitCmp() argument
3114 return DAG.getNode(SystemZISD::ICMP, DL, MVT::i32, C.Op0, C.Op1, in emitCmp()
3115 DAG.getTargetConstant(C.ICmpType, DL, MVT::i32)); in emitCmp()
3119 return DAG.getNode(SystemZISD::TM, DL, MVT::i32, C.Op0, C.Op1, in emitCmp()
3120 DAG.getTargetConstant(RegisterOnly, DL, MVT::i32)); in emitCmp()
3124 SDValue Val = DAG.getNode(C.Opcode, DL, VTs, C.Op0, C.Op1); in emitCmp()
3129 return DAG.getNode(C.Opcode, DL, VTs, C.Chain, C.Op0, C.Op1); in emitCmp()
3131 return DAG.getNode(C.Opcode, DL, MVT::i32, C.Op0, C.Op1); in emitCmp()
3137 static void lowerMUL_LOHI32(SelectionDAG &DAG, const SDLoc &DL, unsigned Extend, in lowerMUL_LOHI32() argument
3140 Op0 = DAG.getNode(Extend, DL, MVT::i64, Op0); in lowerMUL_LOHI32()
3141 Op1 = DAG.getNode(Extend, DL, MVT::i64, Op1); in lowerMUL_LOHI32()
3142 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, Op0, Op1); in lowerMUL_LOHI32()
3143 Hi = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul, in lowerMUL_LOHI32()
3144 DAG.getConstant(32, DL, MVT::i64)); in lowerMUL_LOHI32()
3145 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Hi); in lowerMUL_LOHI32()
3146 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul); in lowerMUL_LOHI32()
3153 static void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT, in lowerGR128Binary() argument
3156 SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, Op0, Op1); in lowerGR128Binary()
3158 Even = DAG.getTargetExtractSubreg(SystemZ::even128(Is32Bit), DL, VT, Result); in lowerGR128Binary()
3159 Odd = DAG.getTargetExtractSubreg(SystemZ::odd128(Is32Bit), DL, VT, Result); in lowerGR128Binary()
3165 static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue CCReg, in emitSETCC() argument
3167 SDValue Ops[] = {DAG.getConstant(1, DL, MVT::i32), in emitSETCC()
3168 DAG.getConstant(0, DL, MVT::i32), in emitSETCC()
3169 DAG.getTargetConstant(CCValid, DL, MVT::i32), in emitSETCC()
3170 DAG.getTargetConstant(CCMask, DL, MVT::i32), CCReg}; in emitSETCC()
3171 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, MVT::i32, Ops); in emitSETCC()
3247 static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL, in expandV4F32ToV2F64() argument
3250 Op = DAG.getVectorShuffle(MVT::v4f32, DL, Op, DAG.getUNDEF(MVT::v4f32), Mask); in expandV4F32ToV2F64()
3253 return DAG.getNode(SystemZISD::STRICT_VEXTEND, DL, VTs, Chain, Op); in expandV4F32ToV2F64()
3255 return DAG.getNode(SystemZISD::VEXTEND, DL, MVT::v2f64, Op); in expandV4F32ToV2F64()
3261 const SDLoc &DL, EVT VT, in getVectorCmp() argument
3270 SDValue H0 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp0, Chain); in getVectorCmp()
3271 SDValue L0 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp0, Chain); in getVectorCmp()
3272 SDValue H1 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp1, Chain); in getVectorCmp()
3273 SDValue L1 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp1, Chain); in getVectorCmp()
3276 SDValue HRes = DAG.getNode(Opcode, DL, VTs, Chain, H0, H1); in getVectorCmp()
3277 SDValue LRes = DAG.getNode(Opcode, DL, VTs, Chain, L0, L1); in getVectorCmp()
3278 SDValue Res = DAG.getNode(SystemZISD::PACK, DL, VT, HRes, LRes); in getVectorCmp()
3282 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); in getVectorCmp()
3284 return DAG.getMergeValues(Ops, DL); in getVectorCmp()
3286 SDValue HRes = DAG.getNode(Opcode, DL, MVT::v2i64, H0, H1); in getVectorCmp()
3287 SDValue LRes = DAG.getNode(Opcode, DL, MVT::v2i64, L0, L1); in getVectorCmp()
3288 return DAG.getNode(SystemZISD::PACK, DL, VT, HRes, LRes); in getVectorCmp()
3292 return DAG.getNode(Opcode, DL, VTs, Chain, CmpOp0, CmpOp1); in getVectorCmp()
3294 return DAG.getNode(Opcode, DL, VT, CmpOp0, CmpOp1); in getVectorCmp()
3302 const SDLoc &DL, EVT VT, in lowerVectorSETCC() argument
3323 DL, VT, CmpOp1, CmpOp0, Chain); in lowerVectorSETCC()
3325 DL, VT, CmpOp0, CmpOp1, Chain); in lowerVectorSETCC()
3326 Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GE); in lowerVectorSETCC()
3328 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in lowerVectorSETCC()
3340 DL, VT, CmpOp1, CmpOp0, Chain); in lowerVectorSETCC()
3342 DL, VT, CmpOp0, CmpOp1, Chain); in lowerVectorSETCC()
3343 Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GT); in lowerVectorSETCC()
3345 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in lowerVectorSETCC()
3355 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0, CmpOp1, Chain); in lowerVectorSETCC()
3359 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp1, CmpOp0, Chain); in lowerVectorSETCC()
3369 DAG.getSplatBuildVector(VT, DL, DAG.getConstant(-1, DL, MVT::i64)); in lowerVectorSETCC()
3370 Cmp = DAG.getNode(ISD::XOR, DL, VT, Cmp, Mask); in lowerVectorSETCC()
3374 Cmp = DAG.getMergeValues(Ops, DL); in lowerVectorSETCC()
3384 SDLoc DL(Op); in lowerSETCC() local
3387 return lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1); in lowerSETCC()
3389 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL)); in lowerSETCC()
3390 SDValue CCReg = emitCmp(DAG, DL, C); in lowerSETCC()
3391 return emitSETCC(DAG, DL, CCReg, C.CCValid, C.CCMask); in lowerSETCC()
3401 SDLoc DL(Op); in lowerSTRICT_FSETCC() local
3404 SDValue Res = lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1, in lowerSTRICT_FSETCC()
3409 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL, Chain, IsSignaling)); in lowerSTRICT_FSETCC()
3410 SDValue CCReg = emitCmp(DAG, DL, C); in lowerSTRICT_FSETCC()
3412 SDValue Result = emitSETCC(DAG, DL, CCReg, C.CCValid, C.CCMask); in lowerSTRICT_FSETCC()
3414 return DAG.getMergeValues(Ops, DL); in lowerSTRICT_FSETCC()
3422 SDLoc DL(Op); in lowerBR_CC() local
3424 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL)); in lowerBR_CC()
3425 SDValue CCReg = emitCmp(DAG, DL, C); in lowerBR_CC()
3427 SystemZISD::BR_CCMASK, DL, Op.getValueType(), Op.getOperand(0), in lowerBR_CC()
3428 DAG.getTargetConstant(C.CCValid, DL, MVT::i32), in lowerBR_CC()
3429 DAG.getTargetConstant(C.CCMask, DL, MVT::i32), Dest, CCReg); in lowerBR_CC()
3443 static SDValue getAbsolute(SelectionDAG &DAG, const SDLoc &DL, SDValue Op, in getAbsolute() argument
3445 Op = DAG.getNode(ISD::ABS, DL, Op.getValueType(), Op); in getAbsolute()
3447 Op = DAG.getNode(ISD::SUB, DL, Op.getValueType(), in getAbsolute()
3448 DAG.getConstant(0, DL, Op.getValueType()), Op); in getAbsolute()
3459 SDLoc DL(Op); in lowerSELECT_CC() local
3461 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL)); in lowerSELECT_CC()
3472 return getAbsolute(DAG, DL, TrueOp, C.CCMask & SystemZ::CCMASK_CMP_LT); in lowerSELECT_CC()
3474 return getAbsolute(DAG, DL, FalseOp, C.CCMask & SystemZ::CCMASK_CMP_GT); in lowerSELECT_CC()
3477 SDValue CCReg = emitCmp(DAG, DL, C); in lowerSELECT_CC()
3479 DAG.getTargetConstant(C.CCValid, DL, MVT::i32), in lowerSELECT_CC()
3480 DAG.getTargetConstant(C.CCMask, DL, MVT::i32), CCReg}; in lowerSELECT_CC()
3482 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, Op.getValueType(), Ops); in lowerSELECT_CC()
3487 SDLoc DL(Node); in lowerGlobalAddress() local
3498 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor); in lowerGlobalAddress()
3499 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); in lowerGlobalAddress()
3506 DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor + Offset); in lowerGlobalAddress()
3507 Result = DAG.getNode(SystemZISD::PCREL_OFFSET, DL, PtrVT, Full, Result); in lowerGlobalAddress()
3513 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT); in lowerGlobalAddress()
3514 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); in lowerGlobalAddress()
3517 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, SystemZII::MO_GOT); in lowerGlobalAddress()
3518 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); in lowerGlobalAddress()
3519 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result, in lowerGlobalAddress()
3522 Result = getADAEntry(DAG, GV, DL, PtrVT); in lowerGlobalAddress()
3529 Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result, in lowerGlobalAddress()
3530 DAG.getConstant(Offset, DL, PtrVT)); in lowerGlobalAddress()
3539 SDLoc DL(Node); in lowerTLSGetOffset() local
3550 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R12D, GOT, Glue); in lowerTLSGetOffset()
3552 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R2D, GOTOffset, Glue); in lowerTLSGetOffset()
3558 Ops.push_back(DAG.getTargetGlobalAddress(Node->getGlobal(), DL, in lowerTLSGetOffset()
3579 Chain = DAG.getNode(Opcode, DL, NodeTys, Ops); in lowerTLSGetOffset()
3583 return DAG.getCopyFromReg(Chain, DL, SystemZ::R2D, PtrVT, Glue); in lowerTLSGetOffset()
3586 SDValue SystemZTargetLowering::lowerThreadPointer(const SDLoc &DL, in lowerThreadPointer() argument
3592 SDValue TPHi = DAG.getCopyFromReg(Chain, DL, SystemZ::A0, MVT::i32); in lowerThreadPointer()
3593 TPHi = DAG.getNode(ISD::ANY_EXTEND, DL, PtrVT, TPHi); in lowerThreadPointer()
3596 SDValue TPLo = DAG.getCopyFromReg(Chain, DL, SystemZ::A1, MVT::i32); in lowerThreadPointer()
3597 TPLo = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TPLo); in lowerThreadPointer()
3600 SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi, in lowerThreadPointer()
3601 DAG.getConstant(32, DL, PtrVT)); in lowerThreadPointer()
3602 return DAG.getNode(ISD::OR, DL, PtrVT, TPHiShifted, TPLo); in lowerThreadPointer()
3609 SDLoc DL(Node); in lowerGlobalTLSAddress() local
3618 SDValue TP = lowerThreadPointer(DL, DAG); in lowerGlobalTLSAddress()
3630 PtrVT, DL, DAG.getEntryNode(), Offset, in lowerGlobalTLSAddress()
3645 PtrVT, DL, DAG.getEntryNode(), Offset, in lowerGlobalTLSAddress()
3663 PtrVT, DL, DAG.getEntryNode(), DTPOffset, in lowerGlobalTLSAddress()
3666 Offset = DAG.getNode(ISD::ADD, DL, PtrVT, Offset, DTPOffset); in lowerGlobalTLSAddress()
3672 Offset = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, in lowerGlobalTLSAddress()
3674 Offset = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Offset); in lowerGlobalTLSAddress()
3676 DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Offset, in lowerGlobalTLSAddress()
3688 PtrVT, DL, DAG.getEntryNode(), Offset, in lowerGlobalTLSAddress()
3695 return DAG.getNode(ISD::ADD, DL, PtrVT, TP, Offset); in lowerGlobalTLSAddress()
3700 SDLoc DL(Node); in lowerBlockAddress() local
3706 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); in lowerBlockAddress()
3712 SDLoc DL(JT); in lowerJumpTable() local
3717 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); in lowerJumpTable()
3722 SDLoc DL(CP); in lowerConstantPool() local
3734 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); in lowerConstantPool()
3744 SDLoc DL(Op); in lowerFRAMEADDR() local
3760 SDValue Offset = DAG.getConstant(TFL->getBackchainOffset(MF), DL, PtrVT); in lowerFRAMEADDR()
3762 BackChain = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), BackChain, in lowerFRAMEADDR()
3764 BackChain = DAG.getNode(ISD::ADD, DL, PtrVT, BackChain, Offset); in lowerFRAMEADDR()
3780 SDLoc DL(Op); in lowerRETURNADDR() local
3793 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, FrameAddr, in lowerRETURNADDR()
3794 DAG.getConstant(Offset, DL, PtrVT)); in lowerRETURNADDR()
3795 return DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Ptr, in lowerRETURNADDR()
3801 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, LinkReg, PtrVT); in lowerRETURNADDR()
3806 SDLoc DL(Op); in lowerBITCAST() local
3816 SDValue NewLoad = DAG.getLoad(ResVT, DL, LoadN->getChain(), in lowerBITCAST()
3826 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, in lowerBITCAST()
3828 In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL, in lowerBITCAST()
3831 In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, In); in lowerBITCAST()
3832 In64 = DAG.getNode(ISD::SHL, DL, MVT::i64, In64, in lowerBITCAST()
3833 DAG.getConstant(32, DL, MVT::i64)); in lowerBITCAST()
3835 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, In64); in lowerBITCAST()
3837 DL, MVT::f32, Out64); in lowerBITCAST()
3840 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64); in lowerBITCAST()
3841 SDValue In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL, in lowerBITCAST()
3843 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, In64); in lowerBITCAST()
3845 return DAG.getTargetExtractSubreg(SystemZ::subreg_h32, DL, in lowerBITCAST()
3847 SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64, in lowerBITCAST()
3848 DAG.getConstant(32, DL, MVT::i64)); in lowerBITCAST()
3849 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Shift); in lowerBITCAST()
3869 SDLoc DL(Op); in lowerVASTART_XPLINK() local
3876 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1), in lowerVASTART_XPLINK()
3890 SDLoc DL(Op); in lowerVASTART_ELF() local
3895 DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), DL, PtrVT), in lowerVASTART_ELF()
3896 DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), DL, PtrVT), in lowerVASTART_ELF()
3907 FieldAddr = DAG.getNode(ISD::ADD, DL, PtrVT, FieldAddr, in lowerVASTART_ELF()
3908 DAG.getIntPtrConstant(Offset, DL)); in lowerVASTART_ELF()
3909 MemOps[I] = DAG.getStore(Chain, DL, Fields[I], FieldAddr, in lowerVASTART_ELF()
3913 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in lowerVASTART_ELF()
3923 SDLoc DL(Op); in lowerVACOPY() local
3927 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(Sz, DL), in lowerVACOPY()
3951 SDLoc DL(Op); in lowerDYNAMIC_STACKALLOC_XPLINK() local
3966 NeededSpace = DAG.getNode(ISD::ADD, DL, PtrVT, NeededSpace, in lowerDYNAMIC_STACKALLOC_XPLINK()
3967 DAG.getConstant(ExtraAlignSpace, DL, PtrVT)); in lowerDYNAMIC_STACKALLOC_XPLINK()
3975 CallingConv::C, IsSigned, DL, DoesNotReturn, in lowerDYNAMIC_STACKALLOC_XPLINK()
3986 SDValue NewSPRegNode = DAG.getCopyFromReg(Chain, DL, SPReg, PtrVT, Glue); in lowerDYNAMIC_STACKALLOC_XPLINK()
3990 SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, PtrMVT); in lowerDYNAMIC_STACKALLOC_XPLINK()
3991 SDValue Result = DAG.getNode(ISD::ADD, DL, PtrMVT, NewSPRegNode, ArgAdjust); in lowerDYNAMIC_STACKALLOC_XPLINK()
3995 Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result, in lowerDYNAMIC_STACKALLOC_XPLINK()
3996 DAG.getConstant(ExtraAlignSpace, DL, PtrVT)); in lowerDYNAMIC_STACKALLOC_XPLINK()
3997 Result = DAG.getNode(ISD::AND, DL, PtrVT, Result, in lowerDYNAMIC_STACKALLOC_XPLINK()
3998 DAG.getConstant(~(RequiredAlign - 1), DL, PtrVT)); in lowerDYNAMIC_STACKALLOC_XPLINK()
4002 return DAG.getMergeValues(Ops, DL); in lowerDYNAMIC_STACKALLOC_XPLINK()
4016 SDLoc DL(Op); in lowerDYNAMIC_STACKALLOC_ELF() local
4030 SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i64); in lowerDYNAMIC_STACKALLOC_ELF()
4035 Backchain = DAG.getLoad(MVT::i64, DL, Chain, getBackchainAddress(OldSP, DAG), in lowerDYNAMIC_STACKALLOC_ELF()
4040 NeededSpace = DAG.getNode(ISD::ADD, DL, MVT::i64, NeededSpace, in lowerDYNAMIC_STACKALLOC_ELF()
4041 DAG.getConstant(ExtraAlignSpace, DL, MVT::i64)); in lowerDYNAMIC_STACKALLOC_ELF()
4046 NewSP = DAG.getNode(SystemZISD::PROBED_ALLOCA, DL, in lowerDYNAMIC_STACKALLOC_ELF()
4051 NewSP = DAG.getNode(ISD::SUB, DL, MVT::i64, OldSP, NeededSpace); in lowerDYNAMIC_STACKALLOC_ELF()
4053 Chain = DAG.getCopyToReg(Chain, DL, SPReg, NewSP); in lowerDYNAMIC_STACKALLOC_ELF()
4059 SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64); in lowerDYNAMIC_STACKALLOC_ELF()
4060 SDValue Result = DAG.getNode(ISD::ADD, DL, MVT::i64, NewSP, ArgAdjust); in lowerDYNAMIC_STACKALLOC_ELF()
4065 DAG.getNode(ISD::ADD, DL, MVT::i64, Result, in lowerDYNAMIC_STACKALLOC_ELF()
4066 DAG.getConstant(ExtraAlignSpace, DL, MVT::i64)); in lowerDYNAMIC_STACKALLOC_ELF()
4068 DAG.getNode(ISD::AND, DL, MVT::i64, Result, in lowerDYNAMIC_STACKALLOC_ELF()
4069 DAG.getConstant(~(RequiredAlign - 1), DL, MVT::i64)); in lowerDYNAMIC_STACKALLOC_ELF()
4073 Chain = DAG.getStore(Chain, DL, Backchain, getBackchainAddress(NewSP, DAG), in lowerDYNAMIC_STACKALLOC_ELF()
4077 return DAG.getMergeValues(Ops, DL); in lowerDYNAMIC_STACKALLOC_ELF()
4082 SDLoc DL(Op); in lowerGET_DYNAMIC_AREA_OFFSET() local
4084 return DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64); in lowerGET_DYNAMIC_AREA_OFFSET()
4090 SDLoc DL(Op); in lowerSMUL_LOHI() local
4095 lowerMUL_LOHI32(DAG, DL, ISD::SIGN_EXTEND, Op.getOperand(0), in lowerSMUL_LOHI()
4101 lowerGR128Binary(DAG, DL, VT, SystemZISD::SMUL_LOHI, in lowerSMUL_LOHI()
4117 SDValue C63 = DAG.getConstant(63, DL, MVT::i64); in lowerSMUL_LOHI()
4120 SDValue LH = DAG.getNode(ISD::SRA, DL, VT, LL, C63); in lowerSMUL_LOHI()
4121 SDValue RH = DAG.getNode(ISD::SRA, DL, VT, RL, C63); in lowerSMUL_LOHI()
4125 lowerGR128Binary(DAG, DL, VT, SystemZISD::UMUL_LOHI, in lowerSMUL_LOHI()
4127 SDValue NegLLTimesRH = DAG.getNode(ISD::AND, DL, VT, LL, RH); in lowerSMUL_LOHI()
4128 SDValue NegLHTimesRL = DAG.getNode(ISD::AND, DL, VT, LH, RL); in lowerSMUL_LOHI()
4129 SDValue NegSum = DAG.getNode(ISD::ADD, DL, VT, NegLLTimesRH, NegLHTimesRL); in lowerSMUL_LOHI()
4130 Ops[1] = DAG.getNode(ISD::SUB, DL, VT, Ops[1], NegSum); in lowerSMUL_LOHI()
4132 return DAG.getMergeValues(Ops, DL); in lowerSMUL_LOHI()
4138 SDLoc DL(Op); in lowerUMUL_LOHI() local
4143 lowerMUL_LOHI32(DAG, DL, ISD::ZERO_EXTEND, Op.getOperand(0), in lowerUMUL_LOHI()
4149 lowerGR128Binary(DAG, DL, VT, SystemZISD::UMUL_LOHI, in lowerUMUL_LOHI()
4151 return DAG.getMergeValues(Ops, DL); in lowerUMUL_LOHI()
4159 SDLoc DL(Op); in lowerSDIVREM() local
4165 Op0 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op0); in lowerSDIVREM()
4167 Op1 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op1); in lowerSDIVREM()
4172 lowerGR128Binary(DAG, DL, VT, SystemZISD::SDIVREM, Op0, Op1, Ops[1], Ops[0]); in lowerSDIVREM()
4173 return DAG.getMergeValues(Ops, DL); in lowerSDIVREM()
4179 SDLoc DL(Op); in lowerUDIVREM() local
4184 lowerGR128Binary(DAG, DL, VT, SystemZISD::UDIVREM, in lowerUDIVREM()
4186 return DAG.getMergeValues(Ops, DL); in lowerUDIVREM()
4238 SDLoc DL(Op); in lowerOR() local
4239 SDValue Low32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LowOp); in lowerOR()
4240 return DAG.getTargetInsertSubreg(SystemZ::subreg_l32, DL, in lowerOR()
4250 SDLoc DL(N); in lowerXALUO() local
4268 SDValue Result = DAG.getNode(BaseOp, DL, MVT::i128, LHS, RHS); in lowerXALUO()
4269 SDValue Flag = DAG.getNode(FlagOp, DL, MVT::i128, LHS, RHS); in lowerXALUO()
4270 Flag = DAG.getNode(ISD::AssertZext, DL, MVT::i128, Flag, in lowerXALUO()
4272 Flag = DAG.getZExtOrTrunc(Flag, DL, N->getValueType(1)); in lowerXALUO()
4274 Flag = DAG.getNode(ISD::XOR, DL, Flag.getValueType(), in lowerXALUO()
4275 Flag, DAG.getConstant(1, DL, Flag.getValueType())); in lowerXALUO()
4276 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, Flag); in lowerXALUO()
4308 SDValue Result = DAG.getNode(BaseOp, DL, VTs, LHS, RHS); in lowerXALUO()
4310 SDValue SetCC = emitSETCC(DAG, DL, Result.getValue(1), CCValid, CCMask); in lowerXALUO()
4312 SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC); in lowerXALUO()
4314 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, SetCC); in lowerXALUO()
4343 SDLoc DL(N); in lowerUADDSUBO_CARRY() local
4362 Carry = DAG.getNode(ISD::XOR, DL, Carry.getValueType(), in lowerUADDSUBO_CARRY()
4363 Carry, DAG.getConstant(1, DL, Carry.getValueType())); in lowerUADDSUBO_CARRY()
4364 Carry = DAG.getZExtOrTrunc(Carry, DL, MVT::i128); in lowerUADDSUBO_CARRY()
4365 SDValue Result = DAG.getNode(BaseOp, DL, MVT::i128, LHS, RHS, Carry); in lowerUADDSUBO_CARRY()
4366 SDValue Flag = DAG.getNode(FlagOp, DL, MVT::i128, LHS, RHS, Carry); in lowerUADDSUBO_CARRY()
4367 Flag = DAG.getNode(ISD::AssertZext, DL, MVT::i128, Flag, in lowerUADDSUBO_CARRY()
4369 Flag = DAG.getZExtOrTrunc(Flag, DL, N->getValueType(1)); in lowerUADDSUBO_CARRY()
4371 Flag = DAG.getNode(ISD::XOR, DL, Flag.getValueType(), in lowerUADDSUBO_CARRY()
4372 Flag, DAG.getConstant(1, DL, Flag.getValueType())); in lowerUADDSUBO_CARRY()
4373 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, Flag); in lowerUADDSUBO_CARRY()
4401 Carry = DAG.getNode(SystemZISD::GET_CCMASK, DL, MVT::i32, Carry, in lowerUADDSUBO_CARRY()
4402 DAG.getConstant(CCValid, DL, MVT::i32), in lowerUADDSUBO_CARRY()
4403 DAG.getConstant(CCMask, DL, MVT::i32)); in lowerUADDSUBO_CARRY()
4406 SDValue Result = DAG.getNode(BaseOp, DL, VTs, LHS, RHS, Carry); in lowerUADDSUBO_CARRY()
4408 SDValue SetCC = emitSETCC(DAG, DL, Result.getValue(1), CCValid, CCMask); in lowerUADDSUBO_CARRY()
4410 SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC); in lowerUADDSUBO_CARRY()
4412 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, SetCC); in lowerUADDSUBO_CARRY()
4418 SDLoc DL(Op); in lowerCTPOP() local
4422 Op = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op); in lowerCTPOP()
4423 Op = DAG.getNode(ISD::CTPOP, DL, MVT::v2i64, Op); in lowerCTPOP()
4424 SDValue Tmp = DAG.getSplatBuildVector(MVT::v2i64, DL, in lowerCTPOP()
4425 DAG.getConstant(0, DL, MVT::i64)); in lowerCTPOP()
4426 Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp); in lowerCTPOP()
4432 Op = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Op); in lowerCTPOP()
4433 Op = DAG.getNode(SystemZISD::POPCNT, DL, MVT::v16i8, Op); in lowerCTPOP()
4438 Op = DAG.getNode(ISD::BITCAST, DL, VT, Op); in lowerCTPOP()
4439 SDValue Shift = DAG.getConstant(8, DL, MVT::i32); in lowerCTPOP()
4440 SDValue Tmp = DAG.getNode(SystemZISD::VSHL_BY_SCALAR, DL, VT, Op, Shift); in lowerCTPOP()
4441 Op = DAG.getNode(ISD::ADD, DL, VT, Op, Tmp); in lowerCTPOP()
4442 Op = DAG.getNode(SystemZISD::VSRL_BY_SCALAR, DL, VT, Op, Shift); in lowerCTPOP()
4446 SDValue Tmp = DAG.getSplatBuildVector(MVT::v16i8, DL, in lowerCTPOP()
4447 DAG.getConstant(0, DL, MVT::i32)); in lowerCTPOP()
4448 Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp); in lowerCTPOP()
4452 SDValue Tmp = DAG.getSplatBuildVector(MVT::v16i8, DL, in lowerCTPOP()
4453 DAG.getConstant(0, DL, MVT::i32)); in lowerCTPOP()
4454 Op = DAG.getNode(SystemZISD::VSUM, DL, MVT::v4i32, Op, Tmp); in lowerCTPOP()
4455 Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp); in lowerCTPOP()
4468 return DAG.getConstant(0, DL, VT); in lowerCTPOP()
4476 Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op); in lowerCTPOP()
4477 Op = DAG.getNode(SystemZISD::POPCNT, DL, MVT::i64, Op); in lowerCTPOP()
4478 Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op); in lowerCTPOP()
4483 SDValue Tmp = DAG.getNode(ISD::SHL, DL, VT, Op, DAG.getConstant(I, DL, VT)); in lowerCTPOP()
4485 Tmp = DAG.getNode(ISD::AND, DL, VT, Tmp, in lowerCTPOP()
4486 DAG.getConstant(((uint64_t)1 << BitSize) - 1, DL, VT)); in lowerCTPOP()
4487 Op = DAG.getNode(ISD::ADD, DL, VT, Op, Tmp); in lowerCTPOP()
4492 Op = DAG.getNode(ISD::SRL, DL, VT, Op, in lowerCTPOP()
4493 DAG.getConstant(BitSize - 8, DL, VT)); in lowerCTPOP()
4500 SDLoc DL(Op); in lowerATOMIC_FENCE() local
4510 return SDValue(DAG.getMachineNode(SystemZ::Serialize, DL, MVT::Other, in lowerATOMIC_FENCE()
4516 return DAG.getNode(ISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0)); in lowerATOMIC_FENCE()
4557 static void getCSAddressAndShifts(SDValue Addr, SelectionDAG &DAG, SDLoc DL, in getCSAddressAndShifts() argument
4564 AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, in getCSAddressAndShifts()
4565 DAG.getConstant(-4, DL, PtrVT)); in getCSAddressAndShifts()
4569 BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, in getCSAddressAndShifts()
4570 DAG.getConstant(3, DL, PtrVT)); in getCSAddressAndShifts()
4571 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); in getCSAddressAndShifts()
4575 NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, in getCSAddressAndShifts()
4576 DAG.getConstant(0, DL, WideVT), BitShift); in getCSAddressAndShifts()
4598 SDLoc DL(Node); in lowerATOMIC_LOAD_OP() local
4604 Src2 = DAG.getConstant(-Const->getSExtValue(), DL, Src2.getValueType()); in lowerATOMIC_LOAD_OP()
4608 getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift); in lowerATOMIC_LOAD_OP()
4616 Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2, in lowerATOMIC_LOAD_OP()
4617 DAG.getConstant(32 - BitSize, DL, WideVT)); in lowerATOMIC_LOAD_OP()
4620 Src2 = DAG.getNode(ISD::OR, DL, WideVT, Src2, in lowerATOMIC_LOAD_OP()
4621 DAG.getConstant(uint32_t(-1) >> BitSize, DL, WideVT)); in lowerATOMIC_LOAD_OP()
4626 DAG.getConstant(BitSize, DL, WideVT) }; in lowerATOMIC_LOAD_OP()
4627 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops, in lowerATOMIC_LOAD_OP()
4632 SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift, in lowerATOMIC_LOAD_OP()
4633 DAG.getConstant(BitSize, DL, WideVT)); in lowerATOMIC_LOAD_OP()
4634 SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift); in lowerATOMIC_LOAD_OP()
4637 return DAG.getMergeValues(RetOps, DL); in lowerATOMIC_LOAD_OP()
4652 SDLoc DL(Src2); in lowerATOMIC_LOAD_SUB() local
4654 DAG.getNode(ISD::SUB, DL, MemVT, DAG.getConstant(0, DL, MemVT), Src2); in lowerATOMIC_LOAD_SUB()
4655 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT, in lowerATOMIC_LOAD_SUB()
4672 SDLoc DL(Node); in lowerATOMIC_CMP_SWAP() local
4678 return DAG.getMergeValues(Results, DL); in lowerATOMIC_CMP_SWAP()
4689 DL, Tys, Ops, NarrowVT, MMO); in lowerATOMIC_CMP_SWAP()
4690 SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1), in lowerATOMIC_CMP_SWAP()
4704 getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift); in lowerATOMIC_CMP_SWAP()
4709 NegBitShift, DAG.getConstant(BitSize, DL, WideVT) }; in lowerATOMIC_CMP_SWAP()
4710 SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAPW, DL, in lowerATOMIC_CMP_SWAP()
4712 SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1), in lowerATOMIC_CMP_SWAP()
4716 SDValue OrigVal = DAG.getNode(ISD::AssertZext, DL, WideVT, AtomicOp.getValue(0), in lowerATOMIC_CMP_SWAP()
4769 SDLoc DL(Op); in lowerSTACKRESTORE() local
4773 Chain, DL, Regs->getStackPointerRegister(), MVT::i64); in lowerSTACKRESTORE()
4774 Backchain = DAG.getLoad(MVT::i64, DL, Chain, getBackchainAddress(OldSP, DAG), in lowerSTACKRESTORE()
4778 Chain = DAG.getCopyToReg(Chain, DL, Regs->getStackPointerRegister(), NewSP); in lowerSTACKRESTORE()
4781 Chain = DAG.getStore(Chain, DL, Backchain, getBackchainAddress(NewSP, DAG), in lowerSTACKRESTORE()
4794 SDLoc DL(Op); in lowerPREFETCH() local
4798 SDValue Ops[] = {Op.getOperand(0), DAG.getTargetConstant(Code, DL, MVT::i32), in lowerPREFETCH()
4800 return DAG.getMemIntrinsicNode(SystemZISD::PREFETCH, DL, in lowerPREFETCH()
4807 SDLoc DL(CCReg); in getCCResult() local
4808 SDValue IPM = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, CCReg); in getCCResult()
4809 return DAG.getNode(ISD::SRL, DL, MVT::i32, IPM, in getCCResult()
4810 DAG.getConstant(SystemZ::IPM_CC, DL, MVT::i32)); in getCCResult()
5161 static SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL, in getPermuteNode() argument
5171 Op0 = DAG.getNode(ISD::BITCAST, DL, InVT, Op0); in getPermuteNode()
5172 Op1 = DAG.getNode(ISD::BITCAST, DL, InVT, Op1); in getPermuteNode()
5175 SDValue Op2 = DAG.getTargetConstant(P.Operand, DL, MVT::i32); in getPermuteNode()
5176 Op = DAG.getNode(SystemZISD::PERMUTE_DWORDS, DL, InVT, Op0, Op1, Op2); in getPermuteNode()
5180 Op = DAG.getNode(SystemZISD::PACK, DL, OutVT, Op0, Op1); in getPermuteNode()
5182 Op = DAG.getNode(P.Opcode, DL, InVT, Op0, Op1); in getPermuteNode()
5207 static SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL, in getGeneralPermuteNode() argument
5211 Ops[I] = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Ops[I]); in getGeneralPermuteNode()
5216 return DAG.getNode(SystemZISD::SHL_DOUBLE, DL, MVT::v16i8, Ops[OpNo0], in getGeneralPermuteNode()
5218 DAG.getTargetConstant(StartIndex, DL, MVT::i32)); in getGeneralPermuteNode()
5248 IndexNodes[I] = DAG.getConstant(ZeroIdx, DL, MVT::i32); in getGeneralPermuteNode()
5251 IndexNodes[I] = DAG.getConstant(BIdx, DL, MVT::i32); in getGeneralPermuteNode()
5256 SDValue Mask = DAG.getBuildVector(MVT::v16i8, DL, IndexNodes); in getGeneralPermuteNode()
5259 return DAG.getNode(SystemZISD::PERMUTE, DL, MVT::v16i8, Mask, Src, in getGeneralPermuteNode()
5262 return DAG.getNode(SystemZISD::PERMUTE, DL, MVT::v16i8, Src, Mask, in getGeneralPermuteNode()
5270 IndexNodes[I] = DAG.getConstant(Bytes[I], DL, MVT::i32); in getGeneralPermuteNode()
5273 SDValue Op2 = DAG.getBuildVector(MVT::v16i8, DL, IndexNodes); in getGeneralPermuteNode()
5274 return DAG.getNode(SystemZISD::PERMUTE, DL, MVT::v16i8, Ops[0], in getGeneralPermuteNode()
5287 SDValue insertUnpackIfPrepared(SelectionDAG &DAG, const SDLoc &DL, SDValue Op);
5378 SDValue GeneralShuffle::getNode(SelectionDAG &DAG, const SDLoc &DL) { in getNode() argument
5420 Ops[I] = getPermuteNode(DAG, DL, *P, SubOps[0], SubOps[1]); in getNode()
5432 Ops[I] = getGeneralPermuteNode(DAG, DL, SubOps, NewBytes); in getNode()
5455 Op = getPermuteNode(DAG, DL, *P, Ops[OpNo0], Ops[OpNo1]); in getNode()
5457 Op = getGeneralPermuteNode(DAG, DL, &Ops[0], Bytes); in getNode()
5459 Op = insertUnpackIfPrepared(DAG, DL, Op); in getNode()
5461 return DAG.getNode(ISD::BITCAST, DL, VT, Op); in getNode()
5549 const SDLoc &DL, in insertUnpackIfPrepared() argument
5556 SDValue PackedOp = DAG.getNode(ISD::BITCAST, DL, InVT, Op); in insertUnpackIfPrepared()
5560 return DAG.getNode(SystemZISD::UNPACKL_HIGH, DL, OutVT, PackedOp); in insertUnpackIfPrepared()
5573 static SDValue buildScalarToVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT, in buildScalarToVector() argument
5580 return DAG.getBuildVector(VT, DL, Ops); in buildScalarToVector()
5584 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Value); in buildScalarToVector()
5589 static SDValue buildMergeScalars(SelectionDAG &DAG, const SDLoc &DL, EVT VT, in buildMergeScalars() argument
5594 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op1); in buildMergeScalars()
5597 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op0); in buildMergeScalars()
5598 return DAG.getNode(SystemZISD::MERGE_HIGH, DL, VT, in buildMergeScalars()
5599 buildScalarToVector(DAG, DL, VT, Op0), in buildMergeScalars()
5600 buildScalarToVector(DAG, DL, VT, Op1)); in buildMergeScalars()
5605 static SDValue joinDwords(SelectionDAG &DAG, const SDLoc &DL, SDValue Op0, in joinDwords() argument
5612 Op0 = Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op1); in joinDwords()
5614 Op0 = Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0); in joinDwords()
5616 Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0); in joinDwords()
5617 Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op1); in joinDwords()
5619 return DAG.getNode(SystemZISD::JOIN_DWORDS, DL, MVT::v2i64, Op0, Op1); in joinDwords()
5685 SystemZTargetLowering::buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT, in buildVector() argument
5716 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Single); in buildVector()
5728 return joinDwords(DAG, DL, Elems[0], Elems[1]); in buildVector()
5732 return buildMergeScalars(DAG, DL, VT, Elems[0], Elems[1]); in buildVector()
5742 SDValue Op01 = buildMergeScalars(DAG, DL, VT, Elems[0], Elems[1]); in buildVector()
5743 SDValue Op23 = buildMergeScalars(DAG, DL, VT, Elems[2], Elems[3]); in buildVector()
5752 Op01 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op01); in buildVector()
5753 Op23 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op23); in buildVector()
5755 DL, MVT::v2i64, Op01, Op23); in buildVector()
5756 return DAG.getNode(ISD::BITCAST, DL, VT, Op); in buildVector()
5782 Result = DAG.getBuildVector(VT, DL, Constants); in buildVector()
5801 Result = DAG.getNode(SystemZISD::REPLICATE, DL, VT, ReplicatedVal); in buildVector()
5811 Result = DAG.getNode(ISD::BITCAST, DL, VT, in buildVector()
5812 joinDwords(DAG, DL, Elem1, Elem2)); in buildVector()
5823 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Result, Elems[I], in buildVector()
5824 DAG.getConstant(I, DL, MVT::i32)); in buildVector()
5831 SDLoc DL(Op); in lowerBUILD_VECTOR() local
5848 return buildScalarToVector(DAG, DL, VT, Op.getOperand(0)); in lowerBUILD_VECTOR()
5855 return buildVector(DAG, DL, VT, Ops); in lowerBUILD_VECTOR()
5861 SDLoc DL(Op); in lowerVECTOR_SHUFFLE() local
5873 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op0.getOperand(Index)); in lowerVECTOR_SHUFFLE()
5875 return DAG.getNode(SystemZISD::SPLAT, DL, VT, Op.getOperand(0), in lowerVECTOR_SHUFFLE()
5876 DAG.getTargetConstant(Index, DL, MVT::i32)); in lowerVECTOR_SHUFFLE()
5893 SDLoc DL(Op); in lowerSCALAR_TO_VECTOR() local
5895 return DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, in lowerSCALAR_TO_VECTOR()
5897 Op.getOperand(0), DAG.getConstant(0, DL, MVT::i32)); in lowerSCALAR_TO_VECTOR()
5903 SDLoc DL(Op); in lowerINSERT_VECTOR_ELT() local
5925 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntVecVT, in lowerINSERT_VECTOR_ELT()
5926 DAG.getNode(ISD::BITCAST, DL, IntVecVT, Op0), in lowerINSERT_VECTOR_ELT()
5927 DAG.getNode(ISD::BITCAST, DL, IntVT, Op1), Op2); in lowerINSERT_VECTOR_ELT()
5928 return DAG.getNode(ISD::BITCAST, DL, VT, Res); in lowerINSERT_VECTOR_ELT()
5935 SDLoc DL(Op); in lowerEXTRACT_VECTOR_ELT() local
5952 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntVT, in lowerEXTRACT_VECTOR_ELT()
5953 DAG.getNode(ISD::BITCAST, DL, IntVecVT, Op0), Op1); in lowerEXTRACT_VECTOR_ELT()
5954 return DAG.getNode(ISD::BITCAST, DL, VT, Res); in lowerEXTRACT_VECTOR_ELT()
5978 SDLoc DL(Op); in lowerZERO_EXTEND_VECTOR_INREG() local
5986 DAG.getSplatVector(InVT, DL, DAG.getConstant(0, DL, InVT.getScalarType())); in lowerZERO_EXTEND_VECTOR_INREG()
5997 SDValue Shuf = DAG.getVectorShuffle(InVT, DL, PackedOp, ZeroVec, Mask); in lowerZERO_EXTEND_VECTOR_INREG()
5998 return DAG.getNode(ISD::BITCAST, DL, OutVT, Shuf); in lowerZERO_EXTEND_VECTOR_INREG()
6006 SDLoc DL(Op); in lowerShift() local
6021 DL, MVT::i32); in lowerShift()
6022 return DAG.getNode(ByScalar, DL, VT, Op0, Shift); in lowerShift()
6030 SDValue Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Splat); in lowerShift()
6031 return DAG.getNode(ByScalar, DL, VT, Op0, Shift); in lowerShift()
6047 SDValue Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, in lowerShift()
6049 return DAG.getNode(ByScalar, DL, VT, Op0, Shift); in lowerShift()
6060 SDLoc DL(Op); in lowerIS_FPCLASS() local
6086 SDValue TDCMaskV = DAG.getConstant(TDCMask, DL, MVT::i64); in lowerIS_FPCLASS()
6088 SDValue Intr = DAG.getNode(SystemZISD::TDC, DL, ResultVT, Arg, TDCMaskV); in lowerIS_FPCLASS()
6228 SDLoc DL(N); in LowerOperationWrapper() local
6233 DL, Tys, Ops, MVT::i128, MMO); in LowerOperationWrapper()
6239 SDLoc DL(N); in LowerOperationWrapper() local
6245 DL, Tys, Ops, MVT::i128, MMO); in LowerOperationWrapper()
6250 Res = SDValue(DAG.getMachineNode(SystemZ::Serialize, DL, in LowerOperationWrapper()
6256 SDLoc DL(N); in LowerOperationWrapper() local
6263 DL, Tys, Ops, MVT::i128, MMO); in LowerOperationWrapper()
6264 SDValue Success = emitSETCC(DAG, DL, Res.getValue(1), in LowerOperationWrapper()
6266 Success = DAG.getZExtOrTrunc(Success, DL, N->getValueType(1)); in LowerOperationWrapper()
6276 SDLoc DL(N); in LowerOperationWrapper() local
6279 SDValue VecBC = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Src); in LowerOperationWrapper()
6280 Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, VecBC, in LowerOperationWrapper()
6281 DAG.getConstant(1, DL, MVT::i32)); in LowerOperationWrapper()
6282 Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, VecBC, in LowerOperationWrapper()
6283 DAG.getConstant(0, DL, MVT::i32)); in LowerOperationWrapper()
6288 DL, MVT::f64, Src); in LowerOperationWrapper()
6290 DL, MVT::f64, Src); in LowerOperationWrapper()
6291 Lo = DAG.getNode(ISD::BITCAST, DL, MVT::i64, LoFP); in LowerOperationWrapper()
6292 Hi = DAG.getNode(ISD::BITCAST, DL, MVT::i64, HiFP); in LowerOperationWrapper()
6294 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i128, Lo, Hi)); in LowerOperationWrapper()
6462 SDValue SystemZTargetLowering::combineExtract(const SDLoc &DL, EVT ResVT, in combineExtract() argument
6517 Op = DAG.getNode(ISD::BITCAST, DL, VT, Op); in combineExtract()
6521 Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op); in combineExtract()
6524 Op = DAG.getNode(ISD::BITCAST, DL, ResVT, Op); in combineExtract()
6557 Op = DAG.getNode(ISD::BITCAST, DL, VecVT, Op); in combineExtract()
6560 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Op, in combineExtract()
6561 DAG.getConstant(Index, DL, MVT::i32)); in combineExtract()
6569 const SDLoc &DL, EVT TruncVT, SDValue Op, DAGCombinerInfo &DCI) const { in combineTruncateExtract() argument
6595 return combineExtract(DL, ResVT, VecVT, Vec, NewIndex, DCI, true); in combineTruncateExtract()
6613 SDLoc DL(N0); in combineZERO_EXTEND() local
6614 SDValue Ops[] = { DAG.getConstant(TrueOp->getZExtValue(), DL, VT), in combineZERO_EXTEND()
6615 DAG.getConstant(FalseOp->getZExtValue(), DL, VT), in combineZERO_EXTEND()
6617 SDValue NewSelect = DAG.getNode(SystemZISD::SELECT_CCMASK, DL, VT, Ops); in combineZERO_EXTEND()
6621 DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), NewSelect); in combineZERO_EXTEND()
6663 SDLoc DL(N0); in combineSIGN_EXTEND_INREG() local
6665 DAG.getConstant(-1, DL, VT), DAG.getConstant(0, DL, VT), in combineSIGN_EXTEND_INREG()
6667 return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops); in combineSIGN_EXTEND_INREG()
6740 SDLoc DL(N); in combineLOAD() local
6785 DAG.getMemBasePlusOffset(LD->getBasePtr(), TypeSize::getFixed(Offset), DL); in combineLOAD()
6787 DAG.getLoad(User->getValueType(0), DL, LD->getChain(), Ptr, in combineLOAD()
6797 SDValue Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in combineLOAD()
6827 SDValue Extract0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, LdVT, in combineLOAD()
6828 Replicate, DAG.getConstant(0, DL, MVT::i32)); in combineLOAD()
6970 SDLoc DL(SN); in combineSTORE() local
6972 DAG.getStore(SN->getChain(), DL, HiPart, SN->getBasePtr(), in combineSTORE()
6976 DAG.getStore(SN->getChain(), DL, LoPart, in combineSTORE()
6977 DAG.getObjectPtrOffset(DL, SN->getBasePtr(), in combineSTORE()
6983 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chain0, Chain1); in combineSTORE()
8128 DebugLoc DL = MI->getDebugLoc(); in createPHIsForSelects() local
8129 BuildMI(*SinkMBB, SinkInsertionPoint, DL, TII->get(SystemZ::PHI), DestReg) in createPHIsForSelects()
8246 DebugLoc DL = MI.getDebugLoc(); in emitCondStore() local
8266 BuildMI(*MBB, MI, DL, TII->get(STOCOpcode)) in emitCondStore()
8297 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) in emitCondStore()
8306 BuildMI(MBB, DL, TII->get(StoreOpcode)) in emitCondStore()
8402 DebugLoc DL = MI.getDebugLoc(); in emitAtomicLoadBinary() local
8426 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal).add(Base).addImm(Disp).addReg(0); in emitAtomicLoadBinary()
8438 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) in emitAtomicLoadBinary()
8441 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) in emitAtomicLoadBinary()
8446 BuildMI(MBB, DL, TII->get(BinOpcode), Tmp).addReg(RotatedOldVal).add(Src2); in emitAtomicLoadBinary()
8448 BuildMI(MBB, DL, TII->get(SystemZ::XILF), RotatedNewVal) in emitAtomicLoadBinary()
8452 BuildMI(MBB, DL, TII->get(BinOpcode), RotatedNewVal) in emitAtomicLoadBinary()
8458 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedNewVal) in emitAtomicLoadBinary()
8461 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) in emitAtomicLoadBinary()
8463 BuildMI(MBB, DL, TII->get(CSOpcode), Dest) in emitAtomicLoadBinary()
8468 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) in emitAtomicLoadBinary()
8497 DebugLoc DL = MI.getDebugLoc(); in emitAtomicLoadMinMax() local
8524 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal).add(Base).addImm(Disp).addReg(0); in emitAtomicLoadMinMax()
8533 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) in emitAtomicLoadMinMax()
8536 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) in emitAtomicLoadMinMax()
8538 BuildMI(MBB, DL, TII->get(CompareOpcode)) in emitAtomicLoadMinMax()
8540 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) in emitAtomicLoadMinMax()
8549 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal) in emitAtomicLoadMinMax()
8562 BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal) in emitAtomicLoadMinMax()
8565 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) in emitAtomicLoadMinMax()
8567 BuildMI(MBB, DL, TII->get(CSOpcode), Dest) in emitAtomicLoadMinMax()
8572 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) in emitAtomicLoadMinMax()
8599 DebugLoc DL = MI.getDebugLoc(); in emitAtomicCmpSwapW() local
8629 BuildMI(MBB, DL, TII->get(LOpcode), OrigOldVal) in emitAtomicCmpSwapW()
8649 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) in emitAtomicCmpSwapW()
8652 BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal) in emitAtomicCmpSwapW()
8655 BuildMI(MBB, DL, TII->get(SystemZ::RLL), OldValRot) in emitAtomicCmpSwapW()
8657 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal) in emitAtomicCmpSwapW()
8659 BuildMI(MBB, DL, TII->get(ZExtOpcode), Dest) in emitAtomicCmpSwapW()
8661 BuildMI(MBB, DL, TII->get(SystemZ::CR)) in emitAtomicCmpSwapW()
8663 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) in emitAtomicCmpSwapW()
8676 BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal) in emitAtomicCmpSwapW()
8678 BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal) in emitAtomicCmpSwapW()
8683 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) in emitAtomicCmpSwapW()
8705 DebugLoc DL = MI.getDebugLoc(); in emitPair128() local
8713 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), Tmp1); in emitPair128()
8714 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Tmp2) in emitPair128()
8716 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest) in emitPair128()
8732 DebugLoc DL = MI.getDebugLoc(); in emitExt128() local
8738 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), In128); in emitExt128()
8743 BuildMI(*MBB, MI, DL, TII->get(SystemZ::LLILL), Zero64) in emitExt128()
8745 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), NewIn128) in emitExt128()
8749 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest) in emitExt128()
8763 DebugLoc DL = MI.getDebugLoc(); in emitMemMemWrapper() local
8805 BuildMI(*InsMBB, InsPos, DL, TII->get(SystemZ::MVI)) in emitMemMemWrapper()
8808 BuildMI(*InsMBB, InsPos, DL, TII->get(SystemZ::STC)) in emitMemMemWrapper()
8813 BuildMI(*MBB, InsPos, DL, TII->get(Opcode)) in emitMemMemWrapper()
8865 BuildMI(*MBB, MI, DL, TII->get(SystemZ::SRLG), StartCountReg) in emitMemMemWrapper()
8874 BuildMI(*MBB, MI, DL, TII->get(SystemZ::LGHI), Reg).addImm(0); in emitMemMemWrapper()
8912 BuildMI(MBB, DL, TII->get(SystemZ::CGHI)) in emitMemMemWrapper()
8914 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) in emitMemMemWrapper()
8928 BuildMI(MBB, DL, TII->get(SystemZ::CGHI)) in emitMemMemWrapper()
8930 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) in emitMemMemWrapper()
8943 BuildMI(MBB, DL, TII->get(SystemZ::J)).addMBB(AllDoneMBB); in emitMemMemWrapper()
8950 BuildMI(MBB, DL, TII->get(SystemZ::CGHI)) in emitMemMemWrapper()
8952 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) in emitMemMemWrapper()
8989 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisDestReg) in emitMemMemWrapper()
8993 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisSrcReg) in emitMemMemWrapper()
8996 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisCountReg) in emitMemMemWrapper()
9000 BuildMI(MBB, DL, TII->get(SystemZ::PFD)) in emitMemMemWrapper()
9007 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) in emitMemMemWrapper()
9024 BuildMI(MBB, DL, TII->get(SystemZ::LA), NextDestReg) in emitMemMemWrapper()
9027 BuildMI(MBB, DL, TII->get(SystemZ::LA), NextSrcReg) in emitMemMemWrapper()
9029 BuildMI(MBB, DL, TII->get(SystemZ::AGHI), NextCountReg) in emitMemMemWrapper()
9031 BuildMI(MBB, DL, TII->get(SystemZ::CGHI)) in emitMemMemWrapper()
9033 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) in emitMemMemWrapper()
9051 BuildMI(MBB, DL, TII->get(SystemZ::PHI), RemDestReg) in emitMemMemWrapper()
9055 BuildMI(MBB, DL, TII->get(SystemZ::PHI), RemSrcReg) in emitMemMemWrapper()
9063 BuildMI(MBB, DL, TII->get(SystemZ::EXRL_Pseudo)) in emitMemMemWrapper()
9094 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) in emitMemMemWrapper()
9119 DebugLoc DL = MI.getDebugLoc(); in emitStringWrapper() local
9150 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This1Reg) in emitStringWrapper()
9153 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This2Reg) in emitStringWrapper()
9156 BuildMI(MBB, DL, TII->get(TargetOpcode::COPY), SystemZ::R0L).addReg(CharReg); in emitStringWrapper()
9157 BuildMI(MBB, DL, TII->get(Opcode)) in emitStringWrapper()
9160 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) in emitStringWrapper()
9223 DebugLoc DL = MI.getDebugLoc(); in emitLoadAndTestCmp0() local
9233 BuildMI(*MBB, MI, DL, TII->get(Opcode), DstReg) in emitLoadAndTestCmp0()
9246 DebugLoc DL = MI.getDebugLoc(); in emitProbedAlloca() local
9269 BuildMI(MBB, DL, TII->get(SystemZ::PHI), PHIReg) in emitProbedAlloca()
9274 BuildMI(MBB, DL, TII->get(SystemZ::CLGFI)) in emitProbedAlloca()
9277 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) in emitProbedAlloca()
9286 BuildMI(MBB, DL, TII->get(SystemZ::SLGFI), IncReg) in emitProbedAlloca()
9289 BuildMI(MBB, DL, TII->get(SystemZ::SLGFI), SystemZ::R15D) in emitProbedAlloca()
9292 BuildMI(MBB, DL, TII->get(SystemZ::CG)).addReg(SystemZ::R15D) in emitProbedAlloca()
9295 BuildMI(MBB, DL, TII->get(SystemZ::J)).addMBB(LoopTestMBB); in emitProbedAlloca()
9302 BuildMI(MBB, DL, TII->get(SystemZ::CGHI)) in emitProbedAlloca()
9305 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) in emitProbedAlloca()
9314 BuildMI(MBB, DL, TII->get(SystemZ::SLGR), SystemZ::R15D) in emitProbedAlloca()
9317 BuildMI(MBB, DL, TII->get(SystemZ::CG)).addReg(SystemZ::R15D) in emitProbedAlloca()
9324 BuildMI(*MBB, MBB->begin(), DL, TII->get(TargetOpcode::COPY), DstReg) in emitProbedAlloca()
9335 SDLoc DL(SP); in getBackchainAddress() local
9336 return DAG.getNode(ISD::ADD, DL, MVT::i64, SP, in getBackchainAddress()
9337 DAG.getIntPtrConstant(TFL->getBackchainOffset(MF), DL)); in getBackchainAddress()