1//===-- AMDGPUGIsel.td - AMDGPU GlobalISel Patterns---------*- tablegen -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// This files contains patterns that should only be used by GlobalISel.  For
9// example patterns for V_* instructions that have S_* equivalents.
10// SelectionDAG does not support selecting V_* instructions.
11//===----------------------------------------------------------------------===//
12
13include "AMDGPU.td"
14include "AMDGPUCombine.td"
15
16def sd_vsrc0 : ComplexPattern<i32, 1, "">;
17def gi_vsrc0 :
18    GIComplexOperandMatcher<s32, "selectVSRC0">,
19    GIComplexPatternEquiv<sd_vsrc0>;
20
21def sd_vcsrc : ComplexPattern<i32, 1, "">;
22def gi_vcsrc :
23    GIComplexOperandMatcher<s32, "selectVCSRC">,
24    GIComplexPatternEquiv<sd_vcsrc>;
25
26def gi_vop3mods0 :
27    GIComplexOperandMatcher<s32, "selectVOP3Mods0">,
28    GIComplexPatternEquiv<VOP3Mods0>;
29
30def gi_vop3mods :
31    GIComplexOperandMatcher<s32, "selectVOP3Mods">,
32    GIComplexPatternEquiv<VOP3Mods>;
33
34def gi_vop3_no_mods :
35    GIComplexOperandMatcher<s32, "selectVOP3NoMods">,
36    GIComplexPatternEquiv<VOP3NoMods>;
37
38def gi_vop3mods_nnan :
39    GIComplexOperandMatcher<s32, "selectVOP3Mods_nnan">,
40    GIComplexPatternEquiv<VOP3Mods_nnan>;
41
42def gi_vop3omods :
43    GIComplexOperandMatcher<s32, "selectVOP3OMods">,
44    GIComplexPatternEquiv<VOP3OMods>;
45
46def gi_vop3opselmods0 :
47    GIComplexOperandMatcher<s32, "selectVOP3OpSelMods0">,
48    GIComplexPatternEquiv<VOP3OpSelMods0>;
49
50def gi_vop3opselmods :
51    GIComplexOperandMatcher<s32, "selectVOP3OpSelMods">,
52    GIComplexPatternEquiv<VOP3OpSelMods>;
53
54def gi_smrd_imm :
55    GIComplexOperandMatcher<s64, "selectSmrdImm">,
56    GIComplexPatternEquiv<SMRDImm>;
57
58def gi_smrd_imm32 :
59    GIComplexOperandMatcher<s64, "selectSmrdImm32">,
60    GIComplexPatternEquiv<SMRDImm32>;
61
62def gi_smrd_sgpr :
63    GIComplexOperandMatcher<s64, "selectSmrdSgpr">,
64    GIComplexPatternEquiv<SMRDSgpr>;
65
66// FIXME: Why are the atomic versions separated?
67def gi_flat_offset :
68    GIComplexOperandMatcher<s64, "selectFlatOffset">,
69    GIComplexPatternEquiv<FLATOffset>;
70def gi_flat_offset_signed :
71    GIComplexOperandMatcher<s64, "selectFlatOffsetSigned">,
72    GIComplexPatternEquiv<FLATOffsetSigned>;
73def gi_flat_atomic :
74    GIComplexOperandMatcher<s64, "selectFlatOffset">,
75    GIComplexPatternEquiv<FLATAtomic>;
76def gi_flat_signed_atomic :
77    GIComplexOperandMatcher<s64, "selectFlatOffsetSigned">,
78    GIComplexPatternEquiv<FLATSignedAtomic>;
79
80def gi_mubuf_scratch_offset :
81    GIComplexOperandMatcher<s32, "selectMUBUFScratchOffset">,
82    GIComplexPatternEquiv<MUBUFScratchOffset>;
83def gi_mubuf_scratch_offen :
84    GIComplexOperandMatcher<s32, "selectMUBUFScratchOffen">,
85    GIComplexPatternEquiv<MUBUFScratchOffen>;
86
87def gi_ds_1addr_1offset :
88    GIComplexOperandMatcher<s32, "selectDS1Addr1Offset">,
89    GIComplexPatternEquiv<DS1Addr1Offset>;
90
91def gi_ds_64bit_4byte_aligned :
92    GIComplexOperandMatcher<s64, "selectDS64Bit4ByteAligned">,
93    GIComplexPatternEquiv<DS64Bit4ByteAligned>;
94
95def gi_mubuf_addr64 :
96    GIComplexOperandMatcher<s64, "selectMUBUFAddr64">,
97    GIComplexPatternEquiv<MUBUFAddr64>;
98
99def gi_mubuf_offset :
100    GIComplexOperandMatcher<s64, "selectMUBUFOffset">,
101    GIComplexPatternEquiv<MUBUFOffset>;
102
103def gi_mubuf_addr64_atomic :
104    GIComplexOperandMatcher<s64, "selectMUBUFAddr64Atomic">,
105    GIComplexPatternEquiv<MUBUFAddr64Atomic>;
106
107def gi_mubuf_offset_atomic :
108    GIComplexOperandMatcher<s64, "selectMUBUFOffsetAtomic">,
109    GIComplexPatternEquiv<MUBUFOffsetAtomic>;
110
111// Separate load nodes are defined to glue m0 initialization in
112// SelectionDAG. The GISel selector can just insert m0 initialization
113// directly before before selecting a glue-less load, so hide this
114// distinction.
115
116def : GINodeEquiv<G_LOAD, AMDGPUld_glue> {
117  let CheckMMOIsNonAtomic = 1;
118}
119
120def : GINodeEquiv<G_STORE, AMDGPUst_glue> {
121  let CheckMMOIsNonAtomic = 1;
122}
123
124def : GINodeEquiv<G_LOAD, AMDGPUatomic_ld_glue> {
125  bit CheckMMOIsAtomic = 1;
126}
127
128
129
130def : GINodeEquiv<G_ATOMIC_CMPXCHG, atomic_cmp_swap_glue>;
131def : GINodeEquiv<G_ATOMICRMW_XCHG, atomic_swap_glue>;
132def : GINodeEquiv<G_ATOMICRMW_ADD, atomic_load_add_glue>;
133def : GINodeEquiv<G_ATOMICRMW_SUB, atomic_load_sub_glue>;
134def : GINodeEquiv<G_ATOMICRMW_AND, atomic_load_and_glue>;
135def : GINodeEquiv<G_ATOMICRMW_OR, atomic_load_or_glue>;
136def : GINodeEquiv<G_ATOMICRMW_XOR, atomic_load_xor_glue>;
137def : GINodeEquiv<G_ATOMICRMW_MIN, atomic_load_min_glue>;
138def : GINodeEquiv<G_ATOMICRMW_MAX, atomic_load_max_glue>;
139def : GINodeEquiv<G_ATOMICRMW_UMIN, atomic_load_umin_glue>;
140def : GINodeEquiv<G_ATOMICRMW_UMAX, atomic_load_umax_glue>;
141def : GINodeEquiv<G_ATOMICRMW_FADD, atomic_load_fadd_glue>;
142
143def : GINodeEquiv<G_AMDGPU_FFBH_U32, AMDGPUffbh_u32>;
144def : GINodeEquiv<G_AMDGPU_FMIN_LEGACY, AMDGPUfmin_legacy>;
145def : GINodeEquiv<G_AMDGPU_FMAX_LEGACY, AMDGPUfmax_legacy>;
146
147def : GINodeEquiv<G_AMDGPU_ATOMIC_CMPXCHG, AMDGPUatomic_cmp_swap>;
148def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD, SIbuffer_load>;
149def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_USHORT, SIbuffer_load_ushort>;
150def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_UBYTE, SIbuffer_load_ubyte>;
151def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_SSHORT, SIbuffer_load_short>;
152def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_SBYTE, SIbuffer_load_byte>;
153def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_FORMAT, SIbuffer_load_format>;
154def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_FORMAT_D16, SIbuffer_load_format_d16>;
155def : GINodeEquiv<G_AMDGPU_TBUFFER_LOAD_FORMAT, SItbuffer_load>;
156def : GINodeEquiv<G_AMDGPU_TBUFFER_LOAD_FORMAT_D16, SItbuffer_load_d16>;
157def : GINodeEquiv<G_AMDGPU_BUFFER_STORE, SIbuffer_store>;
158def : GINodeEquiv<G_AMDGPU_BUFFER_STORE_SHORT, SIbuffer_store_short>;
159def : GINodeEquiv<G_AMDGPU_BUFFER_STORE_BYTE, SIbuffer_store_byte>;
160def : GINodeEquiv<G_AMDGPU_BUFFER_STORE_FORMAT, SIbuffer_store_format>;
161def : GINodeEquiv<G_AMDGPU_BUFFER_STORE_FORMAT_D16, SIbuffer_store_format_d16>;
162def : GINodeEquiv<G_AMDGPU_TBUFFER_STORE_FORMAT, SItbuffer_store>;
163def : GINodeEquiv<G_AMDGPU_TBUFFER_STORE_FORMAT_D16, SItbuffer_store_d16>;
164
165// FIXME: Check MMO is atomic
166def : GINodeEquiv<G_AMDGPU_ATOMIC_INC, SIatomic_inc>;
167def : GINodeEquiv<G_AMDGPU_ATOMIC_DEC, SIatomic_dec>;
168def : GINodeEquiv<G_AMDGPU_ATOMIC_INC, atomic_inc_glue>;
169def : GINodeEquiv<G_AMDGPU_ATOMIC_DEC, atomic_dec_glue>;
170
171def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_SWAP, SIbuffer_atomic_swap>;
172def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_ADD, SIbuffer_atomic_add>;
173def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_SUB, SIbuffer_atomic_sub>;
174def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_SMIN, SIbuffer_atomic_smin>;
175def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_UMIN, SIbuffer_atomic_umin>;
176def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_SMAX, SIbuffer_atomic_smax>;
177def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_UMAX, SIbuffer_atomic_umax>;
178def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_AND, SIbuffer_atomic_and>;
179def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_OR, SIbuffer_atomic_or>;
180def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_XOR, SIbuffer_atomic_xor>;
181def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_INC, SIbuffer_atomic_inc>;
182def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_DEC, SIbuffer_atomic_dec>;
183def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_CMPSWAP, SIbuffer_atomic_cmpswap>;
184
185class GISelSop2Pat <
186  SDPatternOperator node,
187  Instruction inst,
188  ValueType dst_vt,
189  ValueType src0_vt = dst_vt, ValueType src1_vt = src0_vt>   : GCNPat <
190
191  (dst_vt (node (src0_vt SReg_32:$src0), (src1_vt SReg_32:$src1))),
192  (inst src0_vt:$src0, src1_vt:$src1)
193>;
194
195class GISelVop2Pat <
196  SDPatternOperator node,
197  Instruction inst,
198  ValueType dst_vt,
199  ValueType src0_vt = dst_vt, ValueType src1_vt = src0_vt>   : GCNPat <
200
201  (dst_vt (node (src0_vt (sd_vsrc0 src0_vt:$src0)), (src1_vt VGPR_32:$src1))),
202  (inst src0_vt:$src0, src1_vt:$src1)
203>;
204
205class GISelVop2CommutePat <
206  SDPatternOperator node,
207  Instruction inst,
208  ValueType dst_vt,
209  ValueType src0_vt = dst_vt, ValueType src1_vt = src0_vt>   : GCNPat <
210
211  (dst_vt (node (src1_vt VGPR_32:$src1), (src0_vt (sd_vsrc0 src0_vt:$src0)))),
212  (inst src0_vt:$src0, src1_vt:$src1)
213>;
214
215class GISelVop3Pat2 <
216  SDPatternOperator node,
217  Instruction inst,
218  ValueType dst_vt,
219  ValueType src0_vt = dst_vt, ValueType src1_vt = src0_vt>   : GCNPat <
220
221  (dst_vt (node (src0_vt (sd_vcsrc src0_vt:$src0)), (src1_vt (sd_vcsrc src1_vt:$src1)))),
222  (inst src0_vt:$src0, src1_vt:$src1)
223>;
224
225class GISelVop3Pat2CommutePat <
226  SDPatternOperator node,
227  Instruction inst,
228  ValueType dst_vt,
229  ValueType src0_vt = dst_vt, ValueType src1_vt = src0_vt>   : GCNPat <
230
231  (dst_vt (node (src0_vt (sd_vcsrc src0_vt:$src0)), (src1_vt (sd_vcsrc src1_vt:$src1)))),
232  (inst src0_vt:$src1, src1_vt:$src0)
233>;
234
235class GISelVop3Pat2ModsPat <
236  SDPatternOperator node,
237  Instruction inst,
238  ValueType dst_vt,
239  ValueType src0_vt = dst_vt, ValueType src1_vt = src0_vt> : GCNPat <
240
241  (dst_vt (node (src0_vt (VOP3Mods0 src0_vt:$src0, i32:$src0_modifiers, i1:$clamp, i32:$omods)),
242                (src1_vt (VOP3Mods src1_vt:$src1, i32:$src1_modifiers)))),
243  (inst i32:$src0_modifiers, src0_vt:$src0,
244        i32:$src1_modifiers, src1_vt:$src1, $clamp, $omods)
245>;
246
247multiclass GISelVop2IntrPat <
248  SDPatternOperator node, Instruction inst,
249  ValueType dst_vt, ValueType src_vt = dst_vt> {
250
251  def : GISelVop2Pat <node, inst, dst_vt, src_vt>;
252
253  // FIXME: Intrinsics aren't marked as commutable, so we need to add an explcit
254  // pattern to handle commuting.  This is another reason why legalizing to a
255  // generic machine instruction may be better that matching the intrinsic
256  // directly.
257  def : GISelVop2CommutePat <node, inst, dst_vt, src_vt>;
258}
259
260// Since GlobalISel is more flexible then SelectionDAG, I think we can get
261// away with adding patterns for integer types and not legalizing all
262// loads and stores to vector types.  This should help simplify the load/store
263// legalization.
264foreach Ty = [i64, p0, p1, p4] in {
265  defm : SMRD_Pattern <"S_LOAD_DWORDX2",  Ty>;
266}
267
268def gi_as_i32timm : GICustomOperandRenderer<"renderTruncTImm32">,
269  GISDNodeXFormEquiv<as_i32timm>;
270
271def gi_as_i16timm : GICustomOperandRenderer<"renderTruncTImm16">,
272  GISDNodeXFormEquiv<as_i16timm>;
273
274def gi_as_i8timm : GICustomOperandRenderer<"renderTruncTImm8">,
275  GISDNodeXFormEquiv<as_i8timm>;
276
277def gi_as_i1timm : GICustomOperandRenderer<"renderTruncTImm1">,
278  GISDNodeXFormEquiv<as_i1timm>;
279
280def gi_NegateImm : GICustomOperandRenderer<"renderNegateImm">,
281  GISDNodeXFormEquiv<NegateImm>;
282
283def gi_bitcast_fpimm_to_i32 : GICustomOperandRenderer<"renderBitcastImm">,
284  GISDNodeXFormEquiv<bitcast_fpimm_to_i32>;
285
286def gi_IMMPopCount : GICustomOperandRenderer<"renderPopcntImm">,
287  GISDNodeXFormEquiv<IMMPopCount>;
288
289def gi_extract_glc : GICustomOperandRenderer<"renderExtractGLC">,
290  GISDNodeXFormEquiv<extract_glc>;
291
292def gi_extract_slc : GICustomOperandRenderer<"renderExtractSLC">,
293  GISDNodeXFormEquiv<extract_slc>;
294
295def gi_extract_dlc : GICustomOperandRenderer<"renderExtractDLC">,
296  GISDNodeXFormEquiv<extract_dlc>;
297
298def gi_extract_swz : GICustomOperandRenderer<"renderExtractSWZ">,
299  GISDNodeXFormEquiv<extract_swz>;
300