1 //===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the AArch64SelectionDAGInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "AArch64TargetMachine.h"
14 using namespace llvm;
15
16 #define DEBUG_TYPE "aarch64-selectiondag-info"
17
EmitMOPS(AArch64ISD::NodeType SDOpcode,SelectionDAG & DAG,const SDLoc & DL,SDValue Chain,SDValue Dst,SDValue SrcOrValue,SDValue Size,Align Alignment,bool isVolatile,MachinePointerInfo DstPtrInfo,MachinePointerInfo SrcPtrInfo) const18 SDValue AArch64SelectionDAGInfo::EmitMOPS(AArch64ISD::NodeType SDOpcode,
19 SelectionDAG &DAG, const SDLoc &DL,
20 SDValue Chain, SDValue Dst,
21 SDValue SrcOrValue, SDValue Size,
22 Align Alignment, bool isVolatile,
23 MachinePointerInfo DstPtrInfo,
24 MachinePointerInfo SrcPtrInfo) const {
25
26 // Get the constant size of the copy/set.
27 uint64_t ConstSize = 0;
28 if (auto *C = dyn_cast<ConstantSDNode>(Size))
29 ConstSize = C->getZExtValue();
30
31 const bool IsSet = SDOpcode == AArch64ISD::MOPS_MEMSET ||
32 SDOpcode == AArch64ISD::MOPS_MEMSET_TAGGING;
33
34 const auto MachineOpcode = [&]() {
35 switch (SDOpcode) {
36 case AArch64ISD::MOPS_MEMSET:
37 return AArch64::MOPSMemorySetPseudo;
38 case AArch64ISD::MOPS_MEMSET_TAGGING:
39 return AArch64::MOPSMemorySetTaggingPseudo;
40 case AArch64ISD::MOPS_MEMCOPY:
41 return AArch64::MOPSMemoryCopyPseudo;
42 case AArch64ISD::MOPS_MEMMOVE:
43 return AArch64::MOPSMemoryMovePseudo;
44 default:
45 llvm_unreachable("Unhandled MOPS ISD Opcode");
46 }
47 }();
48
49 MachineFunction &MF = DAG.getMachineFunction();
50
51 auto Vol =
52 isVolatile ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
53 auto DstFlags = MachineMemOperand::MOStore | Vol;
54 auto *DstOp =
55 MF.getMachineMemOperand(DstPtrInfo, DstFlags, ConstSize, Alignment);
56
57 if (IsSet) {
58 // Extend value to i64, if required.
59 if (SrcOrValue.getValueType() != MVT::i64)
60 SrcOrValue = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, SrcOrValue);
61 SDValue Ops[] = {Dst, Size, SrcOrValue, Chain};
62 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::Other};
63 MachineSDNode *Node = DAG.getMachineNode(MachineOpcode, DL, ResultTys, Ops);
64 DAG.setNodeMemRefs(Node, {DstOp});
65 return SDValue(Node, 2);
66 } else {
67 SDValue Ops[] = {Dst, SrcOrValue, Size, Chain};
68 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::i64, MVT::Other};
69 MachineSDNode *Node = DAG.getMachineNode(MachineOpcode, DL, ResultTys, Ops);
70
71 auto SrcFlags = MachineMemOperand::MOLoad | Vol;
72 auto *SrcOp =
73 MF.getMachineMemOperand(SrcPtrInfo, SrcFlags, ConstSize, Alignment);
74 DAG.setNodeMemRefs(Node, {DstOp, SrcOp});
75 return SDValue(Node, 3);
76 }
77 }
78
EmitTargetCodeForMemcpy(SelectionDAG & DAG,const SDLoc & DL,SDValue Chain,SDValue Dst,SDValue Src,SDValue Size,Align Alignment,bool isVolatile,bool AlwaysInline,MachinePointerInfo DstPtrInfo,MachinePointerInfo SrcPtrInfo) const79 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemcpy(
80 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
81 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
82 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
83 const AArch64Subtarget &STI =
84 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
85 if (STI.hasMOPS())
86 return EmitMOPS(AArch64ISD::MOPS_MEMCOPY, DAG, DL, Chain, Dst, Src, Size,
87 Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);
88 return SDValue();
89 }
90
EmitTargetCodeForMemset(SelectionDAG & DAG,const SDLoc & dl,SDValue Chain,SDValue Dst,SDValue Src,SDValue Size,Align Alignment,bool isVolatile,bool AlwaysInline,MachinePointerInfo DstPtrInfo) const91 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset(
92 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
93 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
94 MachinePointerInfo DstPtrInfo) const {
95 const AArch64Subtarget &STI =
96 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
97
98 if (STI.hasMOPS()) {
99 return EmitMOPS(AArch64ISD::MOPS_MEMSET, DAG, dl, Chain, Dst, Src, Size,
100 Alignment, isVolatile, DstPtrInfo, MachinePointerInfo{});
101 }
102 return SDValue();
103 }
104
EmitTargetCodeForMemmove(SelectionDAG & DAG,const SDLoc & dl,SDValue Chain,SDValue Dst,SDValue Src,SDValue Size,Align Alignment,bool isVolatile,MachinePointerInfo DstPtrInfo,MachinePointerInfo SrcPtrInfo) const105 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemmove(
106 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
107 SDValue Size, Align Alignment, bool isVolatile,
108 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
109 const AArch64Subtarget &STI =
110 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
111 if (STI.hasMOPS()) {
112 return EmitMOPS(AArch64ISD::MOPS_MEMMOVE, DAG, dl, Chain, Dst, Src, Size,
113 Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);
114 }
115 return SDValue();
116 }
117
118 static const int kSetTagLoopThreshold = 176;
119
EmitUnrolledSetTag(SelectionDAG & DAG,const SDLoc & dl,SDValue Chain,SDValue Ptr,uint64_t ObjSize,const MachineMemOperand * BaseMemOperand,bool ZeroData)120 static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl,
121 SDValue Chain, SDValue Ptr, uint64_t ObjSize,
122 const MachineMemOperand *BaseMemOperand,
123 bool ZeroData) {
124 MachineFunction &MF = DAG.getMachineFunction();
125 unsigned ObjSizeScaled = ObjSize / 16;
126
127 SDValue TagSrc = Ptr;
128 if (Ptr.getOpcode() == ISD::FrameIndex) {
129 int FI = cast<FrameIndexSDNode>(Ptr)->getIndex();
130 Ptr = DAG.getTargetFrameIndex(FI, MVT::i64);
131 // A frame index operand may end up as [SP + offset] => it is fine to use SP
132 // register as the tag source.
133 TagSrc = DAG.getRegister(AArch64::SP, MVT::i64);
134 }
135
136 const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG;
137 const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G;
138
139 SmallVector<SDValue, 8> OutChains;
140 unsigned OffsetScaled = 0;
141 while (OffsetScaled < ObjSizeScaled) {
142 if (ObjSizeScaled - OffsetScaled >= 2) {
143 SDValue AddrNode = DAG.getMemBasePlusOffset(
144 Ptr, TypeSize::getFixed(OffsetScaled * 16), dl);
145 SDValue St = DAG.getMemIntrinsicNode(
146 OpCode2, dl, DAG.getVTList(MVT::Other),
147 {Chain, TagSrc, AddrNode},
148 MVT::v4i64,
149 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16 * 2));
150 OffsetScaled += 2;
151 OutChains.push_back(St);
152 continue;
153 }
154
155 if (ObjSizeScaled - OffsetScaled > 0) {
156 SDValue AddrNode = DAG.getMemBasePlusOffset(
157 Ptr, TypeSize::getFixed(OffsetScaled * 16), dl);
158 SDValue St = DAG.getMemIntrinsicNode(
159 OpCode1, dl, DAG.getVTList(MVT::Other),
160 {Chain, TagSrc, AddrNode},
161 MVT::v2i64,
162 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16));
163 OffsetScaled += 1;
164 OutChains.push_back(St);
165 }
166 }
167
168 SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
169 return Res;
170 }
171
EmitTargetCodeForSetTag(SelectionDAG & DAG,const SDLoc & dl,SDValue Chain,SDValue Addr,SDValue Size,MachinePointerInfo DstPtrInfo,bool ZeroData) const172 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForSetTag(
173 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr,
174 SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const {
175 uint64_t ObjSize = Size->getAsZExtVal();
176 assert(ObjSize % 16 == 0);
177
178 MachineFunction &MF = DAG.getMachineFunction();
179 MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand(
180 DstPtrInfo, MachineMemOperand::MOStore, ObjSize, Align(16));
181
182 bool UseSetTagRangeLoop =
183 kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold;
184 if (!UseSetTagRangeLoop)
185 return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand,
186 ZeroData);
187
188 const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other};
189
190 unsigned Opcode;
191 if (Addr.getOpcode() == ISD::FrameIndex) {
192 int FI = cast<FrameIndexSDNode>(Addr)->getIndex();
193 Addr = DAG.getTargetFrameIndex(FI, MVT::i64);
194 Opcode = ZeroData ? AArch64::STZGloop : AArch64::STGloop;
195 } else {
196 Opcode = ZeroData ? AArch64::STZGloop_wback : AArch64::STGloop_wback;
197 }
198 SDValue Ops[] = {DAG.getTargetConstant(ObjSize, dl, MVT::i64), Addr, Chain};
199 SDNode *St = DAG.getMachineNode(Opcode, dl, ResTys, Ops);
200
201 DAG.setNodeMemRefs(cast<MachineSDNode>(St), {BaseMemOperand});
202 return SDValue(St, 2);
203 }
204