Lines Matching refs:AM
207 bool foldOffsetIntoAddress(uint64_t Offset, X86ISelAddressMode &AM);
208 bool matchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM,
210 bool matchWrapper(SDValue N, X86ISelAddressMode &AM);
211 bool matchAddress(SDValue N, X86ISelAddressMode &AM);
212 bool matchVectorAddress(SDValue N, X86ISelAddressMode &AM);
213 bool matchAdd(SDValue &N, X86ISelAddressMode &AM, unsigned Depth);
214 bool matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
216 bool matchVectorAddressRecursively(SDValue N, X86ISelAddressMode &AM,
218 bool matchAddressBase(SDValue N, X86ISelAddressMode &AM);
264 inline void getAddressOperands(X86ISelAddressMode &AM, const SDLoc &DL, in getAddressOperands() argument
268 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase) in getAddressOperands()
270 AM.Base_FrameIndex, TLI->getPointerTy(CurDAG->getDataLayout())); in getAddressOperands()
271 else if (AM.Base_Reg.getNode()) in getAddressOperands()
272 Base = AM.Base_Reg; in getAddressOperands()
276 Scale = getI8Imm(AM.Scale, DL); in getAddressOperands()
279 if (AM.NegateIndex) { in getAddressOperands()
282 AM.IndexReg), 0); in getAddressOperands()
283 AM.IndexReg = Neg; in getAddressOperands()
286 if (AM.IndexReg.getNode()) in getAddressOperands()
287 Index = AM.IndexReg; in getAddressOperands()
293 if (AM.GV) in getAddressOperands()
294 Disp = CurDAG->getTargetGlobalAddress(AM.GV, SDLoc(), in getAddressOperands()
295 MVT::i32, AM.Disp, in getAddressOperands()
296 AM.SymbolFlags); in getAddressOperands()
297 else if (AM.CP) in getAddressOperands()
298 Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, AM.Alignment, in getAddressOperands()
299 AM.Disp, AM.SymbolFlags); in getAddressOperands()
300 else if (AM.ES) { in getAddressOperands()
301 assert(!AM.Disp && "Non-zero displacement is ignored with ES."); in getAddressOperands()
302 Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags); in getAddressOperands()
303 } else if (AM.MCSym) { in getAddressOperands()
304 assert(!AM.Disp && "Non-zero displacement is ignored with MCSym."); in getAddressOperands()
305 assert(AM.SymbolFlags == 0 && "oo"); in getAddressOperands()
306 Disp = CurDAG->getMCSymbol(AM.MCSym, MVT::i32); in getAddressOperands()
307 } else if (AM.JT != -1) { in getAddressOperands()
308 assert(!AM.Disp && "Non-zero displacement is ignored with JT."); in getAddressOperands()
309 Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags); in getAddressOperands()
310 } else if (AM.BlockAddr) in getAddressOperands()
311 Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp, in getAddressOperands()
312 AM.SymbolFlags); in getAddressOperands()
314 Disp = CurDAG->getTargetConstant(AM.Disp, DL, MVT::i32); in getAddressOperands()
316 if (AM.Segment.getNode()) in getAddressOperands()
317 Segment = AM.Segment; in getAddressOperands()
1659 X86ISelAddressMode &AM) { in foldOffsetIntoAddress() argument
1664 int64_t Val = AM.Disp + Offset; in foldOffsetIntoAddress()
1667 if (Val != 0 && (AM.ES || AM.MCSym)) in foldOffsetIntoAddress()
1674 AM.hasSymbolicDisplacement())) in foldOffsetIntoAddress()
1678 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase && in foldOffsetIntoAddress()
1682 AM.Disp = Val; in foldOffsetIntoAddress()
1687 bool X86DAGToDAGISel::matchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM, in matchLoadInAddress() argument
1701 if (C->getSExtValue() == 0 && AM.Segment.getNode() == nullptr && in matchLoadInAddress()
1709 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); in matchLoadInAddress()
1712 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); in matchLoadInAddress()
1726 bool X86DAGToDAGISel::matchWrapper(SDValue N, X86ISelAddressMode &AM) { in matchWrapper() argument
1729 if (AM.hasSymbolicDisplacement()) in matchWrapper()
1752 if (IsRIPRel && AM.hasBaseOrIndexReg()) in matchWrapper()
1756 X86ISelAddressMode Backup = AM; in matchWrapper()
1761 AM.GV = G->getGlobal(); in matchWrapper()
1762 AM.SymbolFlags = G->getTargetFlags(); in matchWrapper()
1765 AM.CP = CP->getConstVal(); in matchWrapper()
1766 AM.Alignment = CP->getAlign(); in matchWrapper()
1767 AM.SymbolFlags = CP->getTargetFlags(); in matchWrapper()
1770 AM.ES = S->getSymbol(); in matchWrapper()
1771 AM.SymbolFlags = S->getTargetFlags(); in matchWrapper()
1773 AM.MCSym = S->getMCSymbol(); in matchWrapper()
1775 AM.JT = J->getIndex(); in matchWrapper()
1776 AM.SymbolFlags = J->getTargetFlags(); in matchWrapper()
1778 AM.BlockAddr = BA->getBlockAddress(); in matchWrapper()
1779 AM.SymbolFlags = BA->getTargetFlags(); in matchWrapper()
1784 if (foldOffsetIntoAddress(Offset, AM)) { in matchWrapper()
1785 AM = Backup; in matchWrapper()
1790 AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64)); in matchWrapper()
1798 bool X86DAGToDAGISel::matchAddress(SDValue N, X86ISelAddressMode &AM) { in matchAddress() argument
1799 if (matchAddressRecursively(N, AM, 0)) in matchAddress()
1807 AM.BaseType == X86ISelAddressMode::RegBase && in matchAddress()
1808 AM.Base_Reg.getNode() != nullptr && AM.IndexReg.getNode() == nullptr) { in matchAddress()
1809 SDValue Save_Base_Reg = AM.Base_Reg; in matchAddress()
1811 AM.Base_Reg = SDValue(); in matchAddress()
1812 if (matchLoadInAddress(LoadN, AM, /*AllowSegmentRegForX32=*/true)) in matchAddress()
1813 AM.Base_Reg = Save_Base_Reg; in matchAddress()
1819 if (AM.Scale == 2 && in matchAddress()
1820 AM.BaseType == X86ISelAddressMode::RegBase && in matchAddress()
1821 AM.Base_Reg.getNode() == nullptr) { in matchAddress()
1822 AM.Base_Reg = AM.IndexReg; in matchAddress()
1823 AM.Scale = 1; in matchAddress()
1834 AM.Scale == 1 && in matchAddress()
1835 AM.BaseType == X86ISelAddressMode::RegBase && in matchAddress()
1836 AM.Base_Reg.getNode() == nullptr && in matchAddress()
1837 AM.IndexReg.getNode() == nullptr && in matchAddress()
1838 AM.SymbolFlags == X86II::MO_NO_FLAG && in matchAddress()
1839 AM.hasSymbolicDisplacement()) in matchAddress()
1840 AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64); in matchAddress()
1847 bool X86DAGToDAGISel::matchAdd(SDValue &N, X86ISelAddressMode &AM, in matchAdd() argument
1853 X86ISelAddressMode Backup = AM; in matchAdd()
1854 if (!matchAddressRecursively(N.getOperand(0), AM, Depth+1) && in matchAdd()
1855 !matchAddressRecursively(Handle.getValue().getOperand(1), AM, Depth+1)) in matchAdd()
1857 AM = Backup; in matchAdd()
1860 if (!matchAddressRecursively(Handle.getValue().getOperand(1), AM, in matchAdd()
1862 !matchAddressRecursively(Handle.getValue().getOperand(0), AM, Depth + 1)) in matchAdd()
1864 AM = Backup; in matchAdd()
1869 if (AM.BaseType == X86ISelAddressMode::RegBase && in matchAdd()
1870 !AM.Base_Reg.getNode() && in matchAdd()
1871 !AM.IndexReg.getNode()) { in matchAdd()
1873 AM.Base_Reg = N.getOperand(0); in matchAdd()
1874 AM.IndexReg = N.getOperand(1); in matchAdd()
1875 AM.Scale = 1; in matchAdd()
1908 X86ISelAddressMode &AM) { in foldMaskAndShiftToExtract() argument
1941 AM.IndexReg = And; in foldMaskAndShiftToExtract()
1942 AM.Scale = (1 << ScaleLog); in foldMaskAndShiftToExtract()
1950 X86ISelAddressMode &AM) { in foldMaskedShiftToScaledMask() argument
2009 AM.Scale = 1 << ShiftAmt; in foldMaskedShiftToScaledMask()
2010 AM.IndexReg = NewAnd; in foldMaskedShiftToScaledMask()
2044 X86ISelAddressMode &AM) { in foldMaskAndShiftToScale() argument
2120 AM.Scale = 1 << AMShiftAmt; in foldMaskAndShiftToScale()
2121 AM.IndexReg = NewSRL; in foldMaskAndShiftToScale()
2131 X86ISelAddressMode &AM, in foldMaskedShiftToBEXTR() argument
2179 AM.Scale = 1 << AMShiftAmt; in foldMaskedShiftToBEXTR()
2180 AM.IndexReg = NewAnd; in foldMaskedShiftToBEXTR()
2184 bool X86DAGToDAGISel::matchAddressRecursively(SDValue N, X86ISelAddressMode &AM, in matchAddressRecursively() argument
2189 AM.dump(CurDAG); in matchAddressRecursively()
2193 return matchAddressBase(N, AM); in matchAddressRecursively()
2198 if (AM.isRIPRelative()) { in matchAddressRecursively()
2202 if (!(AM.ES || AM.MCSym) && AM.JT != -1) in matchAddressRecursively()
2206 if (!foldOffsetIntoAddress(Cst->getSExtValue(), AM)) in matchAddressRecursively()
2214 if (!AM.hasSymbolicDisplacement() && AM.Disp == 0) in matchAddressRecursively()
2217 AM.MCSym = ESNode->getMCSymbol(); in matchAddressRecursively()
2224 if (!foldOffsetIntoAddress(Val, AM)) in matchAddressRecursively()
2231 if (!matchWrapper(N, AM)) in matchAddressRecursively()
2236 if (!matchLoadInAddress(cast<LoadSDNode>(N), AM)) in matchAddressRecursively()
2241 if (AM.BaseType == X86ISelAddressMode::RegBase && in matchAddressRecursively()
2242 AM.Base_Reg.getNode() == nullptr && in matchAddressRecursively()
2243 (!Subtarget->is64Bit() || isDispSafeForFrameIndex(AM.Disp))) { in matchAddressRecursively()
2244 AM.BaseType = X86ISelAddressMode::FrameIndexBase; in matchAddressRecursively()
2245 AM.Base_FrameIndex = cast<FrameIndexSDNode>(N)->getIndex(); in matchAddressRecursively()
2251 if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) in matchAddressRecursively()
2261 AM.Scale = 1 << Val; in matchAddressRecursively()
2268 AM.IndexReg = ShVal.getOperand(0); in matchAddressRecursively()
2271 if (!foldOffsetIntoAddress(Disp, AM)) in matchAddressRecursively()
2275 AM.IndexReg = ShVal; in matchAddressRecursively()
2283 if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) break; in matchAddressRecursively()
2304 if (!foldMaskAndShiftToScale(*CurDAG, N, Mask, N, X, AM)) in matchAddressRecursively()
2317 if (AM.BaseType == X86ISelAddressMode::RegBase && in matchAddressRecursively()
2318 AM.Base_Reg.getNode() == nullptr && in matchAddressRecursively()
2319 AM.IndexReg.getNode() == nullptr) { in matchAddressRecursively()
2323 AM.Scale = unsigned(CN->getZExtValue())-1; in matchAddressRecursively()
2337 if (foldOffsetIntoAddress(Disp, AM)) in matchAddressRecursively()
2343 AM.IndexReg = AM.Base_Reg = Reg; in matchAddressRecursively()
2362 X86ISelAddressMode Backup = AM; in matchAddressRecursively()
2363 if (matchAddressRecursively(N.getOperand(0), AM, Depth+1)) { in matchAddressRecursively()
2365 AM = Backup; in matchAddressRecursively()
2370 if (AM.IndexReg.getNode() || AM.isRIPRelative()) { in matchAddressRecursively()
2371 AM = Backup; in matchAddressRecursively()
2389 if ((AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode() && in matchAddressRecursively()
2390 !AM.Base_Reg.getNode()->hasOneUse()) || in matchAddressRecursively()
2391 AM.BaseType == X86ISelAddressMode::FrameIndexBase) in matchAddressRecursively()
2395 if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) + in matchAddressRecursively()
2396 ((AM.Disp != 0) && (Backup.Disp == 0)) + in matchAddressRecursively()
2397 (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2) in matchAddressRecursively()
2401 AM = Backup; in matchAddressRecursively()
2408 AM.IndexReg = RHS; in matchAddressRecursively()
2409 AM.NegateIndex = true; in matchAddressRecursively()
2410 AM.Scale = 1; in matchAddressRecursively()
2415 if (!matchAdd(N, AM, Depth)) in matchAddressRecursively()
2427 !matchAdd(N, AM, Depth)) in matchAddressRecursively()
2435 if (isMinSignedConstant(N.getOperand(1)) && !matchAdd(N, AM, Depth)) in matchAddressRecursively()
2444 if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) break; in matchAddressRecursively()
2461 if (!foldMaskAndShiftToExtract(*CurDAG, N, Mask, Shift, X, AM)) in matchAddressRecursively()
2465 if (!foldMaskAndShiftToScale(*CurDAG, N, Mask, Shift, X, AM)) in matchAddressRecursively()
2469 if (!foldMaskedShiftToBEXTR(*CurDAG, N, Mask, Shift, X, AM, *Subtarget)) in matchAddressRecursively()
2475 if (!foldMaskedShiftToScaledMask(*CurDAG, N, AM)) in matchAddressRecursively()
2483 if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) in matchAddressRecursively()
2508 AM.Scale = 1 << ShAmtC->getZExtValue(); in matchAddressRecursively()
2509 AM.IndexReg = Zext; in matchAddressRecursively()
2519 return matchAddressBase(N, AM); in matchAddressRecursively()
2524 bool X86DAGToDAGISel::matchAddressBase(SDValue N, X86ISelAddressMode &AM) { in matchAddressBase() argument
2526 if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) { in matchAddressBase()
2528 if (!AM.IndexReg.getNode()) { in matchAddressBase()
2529 AM.IndexReg = N; in matchAddressBase()
2530 AM.Scale = 1; in matchAddressBase()
2539 AM.BaseType = X86ISelAddressMode::RegBase; in matchAddressBase()
2540 AM.Base_Reg = N; in matchAddressBase()
2545 X86ISelAddressMode &AM, in matchVectorAddressRecursively() argument
2550 AM.dump(CurDAG); in matchVectorAddressRecursively()
2554 return matchAddressBase(N, AM); in matchVectorAddressRecursively()
2560 if (!foldOffsetIntoAddress(Val, AM)) in matchVectorAddressRecursively()
2565 if (!matchWrapper(N, AM)) in matchVectorAddressRecursively()
2573 X86ISelAddressMode Backup = AM; in matchVectorAddressRecursively()
2574 if (!matchVectorAddressRecursively(N.getOperand(0), AM, Depth + 1) && in matchVectorAddressRecursively()
2575 !matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM, in matchVectorAddressRecursively()
2578 AM = Backup; in matchVectorAddressRecursively()
2581 if (!matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM, in matchVectorAddressRecursively()
2583 !matchVectorAddressRecursively(Handle.getValue().getOperand(0), AM, in matchVectorAddressRecursively()
2586 AM = Backup; in matchVectorAddressRecursively()
2593 return matchAddressBase(N, AM); in matchVectorAddressRecursively()
2599 bool X86DAGToDAGISel::matchVectorAddress(SDValue N, X86ISelAddressMode &AM) { in matchVectorAddress() argument
2600 return matchVectorAddressRecursively(N, AM, 0); in matchVectorAddress()
2608 X86ISelAddressMode AM; in selectVectorAddr() local
2609 AM.IndexReg = IndexOp; in selectVectorAddr()
2610 AM.Scale = cast<ConstantSDNode>(ScaleOp)->getZExtValue(); in selectVectorAddr()
2614 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); in selectVectorAddr()
2616 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); in selectVectorAddr()
2618 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16); in selectVectorAddr()
2624 if (matchVectorAddress(BasePtr, AM)) in selectVectorAddr()
2627 getAddressOperands(AM, DL, VT, Base, Scale, Index, Disp, Segment); in selectVectorAddr()
2641 X86ISelAddressMode AM; in selectAddr() local
2656 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); in selectAddr()
2658 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); in selectAddr()
2660 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16); in selectAddr()
2667 if (matchAddress(N, AM)) in selectAddr()
2670 getAddressOperands(AM, DL, VT, Base, Scale, Index, Disp, Segment); in selectAddr()
2740 X86ISelAddressMode AM; in selectLEAAddr() local
2748 SDValue Copy = AM.Segment; in selectLEAAddr()
2750 AM.Segment = T; in selectLEAAddr()
2751 if (matchAddress(N, AM)) in selectLEAAddr()
2753 assert (T == AM.Segment); in selectLEAAddr()
2754 AM.Segment = Copy; in selectLEAAddr()
2757 if (AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode()) in selectLEAAddr()
2759 else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase) in selectLEAAddr()
2762 if (AM.IndexReg.getNode()) in selectLEAAddr()
2767 if (AM.Scale > 1) in selectLEAAddr()
2775 if (AM.hasSymbolicDisplacement()) { in selectLEAAddr()
2813 if (AM.Disp) in selectLEAAddr()
2820 getAddressOperands(AM, DL, VT, Base, Scale, Index, Disp, Segment); in selectLEAAddr()
2831 X86ISelAddressMode AM; in selectTLSADDRAddr() local
2832 AM.GV = GA->getGlobal(); in selectTLSADDRAddr()
2833 AM.Disp += GA->getOffset(); in selectTLSADDRAddr()
2834 AM.SymbolFlags = GA->getTargetFlags(); in selectTLSADDRAddr()
2837 AM.Scale = 1; in selectTLSADDRAddr()
2838 AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32); in selectTLSADDRAddr()
2842 getAddressOperands(AM, SDLoc(N), VT, Base, Scale, Index, Disp, Segment); in selectTLSADDRAddr()