1//===-- AMDGPUGIsel.td - AMDGPU GlobalISel Patterns---------*- tablegen -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// This files contains patterns that should only be used by GlobalISel.  For
9// example patterns for V_* instructions that have S_* equivalents.
10// SelectionDAG does not support selecting V_* instructions.
11//===----------------------------------------------------------------------===//
12
13include "AMDGPU.td"
14include "AMDGPUCombine.td"
15
16def sd_vsrc0 : ComplexPattern<i32, 1, "">;
17def gi_vsrc0 :
18    GIComplexOperandMatcher<s32, "selectVSRC0">,
19    GIComplexPatternEquiv<sd_vsrc0>;
20
21def sd_vcsrc : ComplexPattern<i32, 1, "">;
22def gi_vcsrc :
23    GIComplexOperandMatcher<s32, "selectVCSRC">,
24    GIComplexPatternEquiv<sd_vcsrc>;
25
26def gi_vop3mods0 :
27    GIComplexOperandMatcher<s32, "selectVOP3Mods0">,
28    GIComplexPatternEquiv<VOP3Mods0>;
29
30def gi_vop3mods :
31    GIComplexOperandMatcher<s32, "selectVOP3Mods">,
32    GIComplexPatternEquiv<VOP3Mods>;
33
34def gi_vop3_no_mods :
35    GIComplexOperandMatcher<s32, "selectVOP3NoMods">,
36    GIComplexPatternEquiv<VOP3NoMods>;
37
38def gi_vop3mods_nnan :
39    GIComplexOperandMatcher<s32, "selectVOP3Mods_nnan">,
40    GIComplexPatternEquiv<VOP3Mods_nnan>;
41
42def gi_vop3omods :
43    GIComplexOperandMatcher<s32, "selectVOP3OMods">,
44    GIComplexPatternEquiv<VOP3OMods>;
45
46def gi_vop3pmods :
47    GIComplexOperandMatcher<s32, "selectVOP3PMods">,
48    GIComplexPatternEquiv<VOP3PMods>;
49
50def gi_vop3opselmods :
51    GIComplexOperandMatcher<s32, "selectVOP3OpSelMods">,
52    GIComplexPatternEquiv<VOP3OpSelMods>;
53
54// FIXME: Why do we have both VOP3OpSel and VOP3OpSelMods?
55def gi_vop3opsel :
56    GIComplexOperandMatcher<s32, "selectVOP3OpSelMods">,
57    GIComplexPatternEquiv<VOP3OpSel>;
58
59def gi_smrd_imm :
60    GIComplexOperandMatcher<s64, "selectSmrdImm">,
61    GIComplexPatternEquiv<SMRDImm>;
62
63def gi_smrd_imm32 :
64    GIComplexOperandMatcher<s64, "selectSmrdImm32">,
65    GIComplexPatternEquiv<SMRDImm32>;
66
67def gi_smrd_sgpr :
68    GIComplexOperandMatcher<s64, "selectSmrdSgpr">,
69    GIComplexPatternEquiv<SMRDSgpr>;
70
71def gi_flat_offset :
72    GIComplexOperandMatcher<s64, "selectFlatOffset">,
73    GIComplexPatternEquiv<FLATOffset>;
74def gi_flat_offset_signed :
75    GIComplexOperandMatcher<s64, "selectFlatOffsetSigned">,
76    GIComplexPatternEquiv<FLATOffsetSigned>;
77def gi_global_saddr :
78    GIComplexOperandMatcher<s64, "selectGlobalSAddr">,
79    GIComplexPatternEquiv<GlobalSAddr>;
80
81def gi_mubuf_scratch_offset :
82    GIComplexOperandMatcher<s32, "selectMUBUFScratchOffset">,
83    GIComplexPatternEquiv<MUBUFScratchOffset>;
84def gi_mubuf_scratch_offen :
85    GIComplexOperandMatcher<s32, "selectMUBUFScratchOffen">,
86    GIComplexPatternEquiv<MUBUFScratchOffen>;
87
88def gi_flat_scratch_offset :
89    GIComplexOperandMatcher<s32, "selectFlatOffsetSigned">,
90    GIComplexPatternEquiv<ScratchOffset>;
91
92def gi_flat_scratch_saddr :
93    GIComplexOperandMatcher<s32, "selectScratchSAddr">,
94    GIComplexPatternEquiv<ScratchSAddr>;
95
96def gi_ds_1addr_1offset :
97    GIComplexOperandMatcher<s32, "selectDS1Addr1Offset">,
98    GIComplexPatternEquiv<DS1Addr1Offset>;
99
100def gi_ds_64bit_4byte_aligned :
101    GIComplexOperandMatcher<s64, "selectDS64Bit4ByteAligned">,
102    GIComplexPatternEquiv<DS64Bit4ByteAligned>;
103
104def gi_ds_128bit_8byte_aligned :
105    GIComplexOperandMatcher<s64, "selectDS128Bit8ByteAligned">,
106    GIComplexPatternEquiv<DS128Bit8ByteAligned>;
107
108def gi_mubuf_addr64 :
109    GIComplexOperandMatcher<s64, "selectMUBUFAddr64">,
110    GIComplexPatternEquiv<MUBUFAddr64>;
111
112def gi_mubuf_offset :
113    GIComplexOperandMatcher<s64, "selectMUBUFOffset">,
114    GIComplexPatternEquiv<MUBUFOffset>;
115
116def gi_smrd_buffer_imm :
117    GIComplexOperandMatcher<s64, "selectSMRDBufferImm">,
118    GIComplexPatternEquiv<SMRDBufferImm>;
119
120def gi_smrd_buffer_imm32 :
121    GIComplexOperandMatcher<s64, "selectSMRDBufferImm32">,
122    GIComplexPatternEquiv<SMRDBufferImm32>;
123
124// Separate load nodes are defined to glue m0 initialization in
125// SelectionDAG. The GISel selector can just insert m0 initialization
126// directly before before selecting a glue-less load, so hide this
127// distinction.
128
129def : GINodeEquiv<G_LOAD, AMDGPUld_glue> {
130  let CheckMMOIsNonAtomic = 1;
131}
132
133def : GINodeEquiv<G_STORE, AMDGPUst_glue> {
134  let CheckMMOIsNonAtomic = 1;
135}
136
137def : GINodeEquiv<G_LOAD, AMDGPUatomic_ld_glue> {
138  bit CheckMMOIsAtomic = 1;
139}
140
141def : GINodeEquiv<G_STORE, AMDGPUatomic_st_glue> {
142  bit CheckMMOIsAtomic = 1;
143}
144
145
146def : GINodeEquiv<G_ATOMIC_CMPXCHG, atomic_cmp_swap_glue>;
147def : GINodeEquiv<G_ATOMICRMW_XCHG, atomic_swap_glue>;
148def : GINodeEquiv<G_ATOMICRMW_ADD, atomic_load_add_glue>;
149def : GINodeEquiv<G_ATOMICRMW_SUB, atomic_load_sub_glue>;
150def : GINodeEquiv<G_ATOMICRMW_AND, atomic_load_and_glue>;
151def : GINodeEquiv<G_ATOMICRMW_OR, atomic_load_or_glue>;
152def : GINodeEquiv<G_ATOMICRMW_XOR, atomic_load_xor_glue>;
153def : GINodeEquiv<G_ATOMICRMW_MIN, atomic_load_min_glue>;
154def : GINodeEquiv<G_ATOMICRMW_MAX, atomic_load_max_glue>;
155def : GINodeEquiv<G_ATOMICRMW_UMIN, atomic_load_umin_glue>;
156def : GINodeEquiv<G_ATOMICRMW_UMAX, atomic_load_umax_glue>;
157def : GINodeEquiv<G_ATOMICRMW_FADD, atomic_load_fadd_glue>;
158
159def : GINodeEquiv<G_AMDGPU_FFBH_U32, AMDGPUffbh_u32_impl>;
160def : GINodeEquiv<G_AMDGPU_FMIN_LEGACY, AMDGPUfmin_legacy>;
161def : GINodeEquiv<G_AMDGPU_FMAX_LEGACY, AMDGPUfmax_legacy>;
162def : GINodeEquiv<G_AMDGPU_RCP_IFLAG, AMDGPUrcp_iflag>;
163
164def : GINodeEquiv<G_AMDGPU_CVT_F32_UBYTE0, AMDGPUcvt_f32_ubyte0>;
165def : GINodeEquiv<G_AMDGPU_CVT_F32_UBYTE1, AMDGPUcvt_f32_ubyte1>;
166def : GINodeEquiv<G_AMDGPU_CVT_F32_UBYTE2, AMDGPUcvt_f32_ubyte2>;
167def : GINodeEquiv<G_AMDGPU_CVT_F32_UBYTE3, AMDGPUcvt_f32_ubyte3>;
168
169def : GINodeEquiv<G_AMDGPU_CVT_PK_I16_I32, AMDGPUpk_i16_i32_impl>;
170def : GINodeEquiv<G_AMDGPU_MED3, AMDGPUsmed3>;
171
172def : GINodeEquiv<G_AMDGPU_ATOMIC_CMPXCHG, AMDGPUatomic_cmp_swap>;
173def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD, SIbuffer_load>;
174def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_USHORT, SIbuffer_load_ushort>;
175def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_UBYTE, SIbuffer_load_ubyte>;
176def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_SSHORT, SIbuffer_load_short>;
177def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_SBYTE, SIbuffer_load_byte>;
178def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_FORMAT, SIbuffer_load_format>;
179def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_FORMAT_D16, SIbuffer_load_format_d16>;
180def : GINodeEquiv<G_AMDGPU_TBUFFER_LOAD_FORMAT, SItbuffer_load>;
181def : GINodeEquiv<G_AMDGPU_TBUFFER_LOAD_FORMAT_D16, SItbuffer_load_d16>;
182def : GINodeEquiv<G_AMDGPU_BUFFER_STORE, SIbuffer_store>;
183def : GINodeEquiv<G_AMDGPU_BUFFER_STORE_SHORT, SIbuffer_store_short>;
184def : GINodeEquiv<G_AMDGPU_BUFFER_STORE_BYTE, SIbuffer_store_byte>;
185def : GINodeEquiv<G_AMDGPU_BUFFER_STORE_FORMAT, SIbuffer_store_format>;
186def : GINodeEquiv<G_AMDGPU_BUFFER_STORE_FORMAT_D16, SIbuffer_store_format_d16>;
187def : GINodeEquiv<G_AMDGPU_TBUFFER_STORE_FORMAT, SItbuffer_store>;
188def : GINodeEquiv<G_AMDGPU_TBUFFER_STORE_FORMAT_D16, SItbuffer_store_d16>;
189
190// FIXME: Check MMO is atomic
191def : GINodeEquiv<G_AMDGPU_ATOMIC_INC, SIatomic_inc>;
192def : GINodeEquiv<G_AMDGPU_ATOMIC_DEC, SIatomic_dec>;
193def : GINodeEquiv<G_AMDGPU_ATOMIC_INC, atomic_inc_glue>;
194def : GINodeEquiv<G_AMDGPU_ATOMIC_DEC, atomic_dec_glue>;
195def : GINodeEquiv<G_AMDGPU_ATOMIC_FMIN, SIatomic_fmin>;
196def : GINodeEquiv<G_AMDGPU_ATOMIC_FMAX, SIatomic_fmax>;
197def : GINodeEquiv<G_AMDGPU_ATOMIC_FMIN, atomic_load_fmin_glue>;
198def : GINodeEquiv<G_AMDGPU_ATOMIC_FMAX, atomic_load_fmax_glue>;
199
200
201def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_SWAP, SIbuffer_atomic_swap>;
202def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_ADD, SIbuffer_atomic_add>;
203def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_SUB, SIbuffer_atomic_sub>;
204def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_SMIN, SIbuffer_atomic_smin>;
205def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_UMIN, SIbuffer_atomic_umin>;
206def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_SMAX, SIbuffer_atomic_smax>;
207def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_UMAX, SIbuffer_atomic_umax>;
208def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_AND, SIbuffer_atomic_and>;
209def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_OR, SIbuffer_atomic_or>;
210def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_XOR, SIbuffer_atomic_xor>;
211def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_INC, SIbuffer_atomic_inc>;
212def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_DEC, SIbuffer_atomic_dec>;
213def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_FADD, SIbuffer_atomic_fadd>;
214def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_FMIN, SIbuffer_atomic_fmin>;
215def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_FMAX, SIbuffer_atomic_fmax>;
216def : GINodeEquiv<G_AMDGPU_BUFFER_ATOMIC_CMPSWAP, SIbuffer_atomic_cmpswap>;
217def : GINodeEquiv<G_AMDGPU_S_BUFFER_LOAD, SIsbuffer_load>;
218
219class GISelSop2Pat <
220  SDPatternOperator node,
221  Instruction inst,
222  ValueType dst_vt,
223  ValueType src0_vt = dst_vt, ValueType src1_vt = src0_vt>   : GCNPat <
224
225  (dst_vt (node (src0_vt SReg_32:$src0), (src1_vt SReg_32:$src1))),
226  (inst src0_vt:$src0, src1_vt:$src1)
227>;
228
229class GISelVop2Pat <
230  SDPatternOperator node,
231  Instruction inst,
232  ValueType dst_vt,
233  ValueType src0_vt = dst_vt, ValueType src1_vt = src0_vt>   : GCNPat <
234
235  (dst_vt (node (src0_vt (sd_vsrc0 src0_vt:$src0)), (src1_vt VGPR_32:$src1))),
236  (inst src0_vt:$src0, src1_vt:$src1)
237>;
238
239class GISelVop2CommutePat <
240  SDPatternOperator node,
241  Instruction inst,
242  ValueType dst_vt,
243  ValueType src0_vt = dst_vt, ValueType src1_vt = src0_vt>   : GCNPat <
244
245  (dst_vt (node (src1_vt VGPR_32:$src1), (src0_vt (sd_vsrc0 src0_vt:$src0)))),
246  (inst src0_vt:$src0, src1_vt:$src1)
247>;
248
249class GISelVop3Pat2 <
250  SDPatternOperator node,
251  Instruction inst,
252  ValueType dst_vt,
253  ValueType src0_vt = dst_vt, ValueType src1_vt = src0_vt>   : GCNPat <
254
255  (dst_vt (node (src0_vt (sd_vcsrc src0_vt:$src0)), (src1_vt (sd_vcsrc src1_vt:$src1)))),
256  (inst src0_vt:$src0, src1_vt:$src1)
257>;
258
259class GISelVop3Pat2CommutePat <
260  SDPatternOperator node,
261  Instruction inst,
262  ValueType dst_vt,
263  ValueType src0_vt = dst_vt, ValueType src1_vt = src0_vt>   : GCNPat <
264
265  (dst_vt (node (src0_vt (sd_vcsrc src0_vt:$src0)), (src1_vt (sd_vcsrc src1_vt:$src1)))),
266  (inst src0_vt:$src1, src1_vt:$src0)
267>;
268
269class GISelVop3Pat2ModsPat <
270  SDPatternOperator node,
271  Instruction inst,
272  ValueType dst_vt,
273  ValueType src0_vt = dst_vt, ValueType src1_vt = src0_vt> : GCNPat <
274
275  (dst_vt (node (src0_vt (VOP3Mods0 src0_vt:$src0, i32:$src0_modifiers, i1:$clamp, i32:$omods)),
276                (src1_vt (VOP3Mods src1_vt:$src1, i32:$src1_modifiers)))),
277  (inst i32:$src0_modifiers, src0_vt:$src0,
278        i32:$src1_modifiers, src1_vt:$src1, $clamp, $omods)
279>;
280
281multiclass GISelVop2IntrPat <
282  SDPatternOperator node, Instruction inst,
283  ValueType dst_vt, ValueType src_vt = dst_vt> {
284
285  def : GISelVop2Pat <node, inst, dst_vt, src_vt>;
286
287  // FIXME: Intrinsics aren't marked as commutable, so we need to add an explicit
288  // pattern to handle commuting.  This is another reason why legalizing to a
289  // generic machine instruction may be better that matching the intrinsic
290  // directly.
291  def : GISelVop2CommutePat <node, inst, dst_vt, src_vt>;
292}
293
294// Since GlobalISel is more flexible then SelectionDAG, I think we can get
295// away with adding patterns for integer types and not legalizing all
296// loads and stores to vector types.  This should help simplify the load/store
297// legalization.
298foreach Ty = [i64, p0, p1, p4] in {
299  defm : SMRD_Pattern <"S_LOAD_DWORDX2",  Ty>;
300}
301
302def gi_as_i32timm : GICustomOperandRenderer<"renderTruncTImm">,
303  GISDNodeXFormEquiv<as_i32timm>;
304
305def gi_as_i16timm : GICustomOperandRenderer<"renderTruncTImm">,
306  GISDNodeXFormEquiv<as_i16timm>;
307
308def gi_as_i8timm : GICustomOperandRenderer<"renderTruncTImm">,
309  GISDNodeXFormEquiv<as_i8timm>;
310
311def gi_as_i1timm : GICustomOperandRenderer<"renderTruncTImm">,
312  GISDNodeXFormEquiv<as_i1timm>;
313
314def gi_NegateImm : GICustomOperandRenderer<"renderNegateImm">,
315  GISDNodeXFormEquiv<NegateImm>;
316
317def gi_bitcast_fpimm_to_i32 : GICustomOperandRenderer<"renderBitcastImm">,
318  GISDNodeXFormEquiv<bitcast_fpimm_to_i32>;
319
320def gi_IMMPopCount : GICustomOperandRenderer<"renderPopcntImm">,
321  GISDNodeXFormEquiv<IMMPopCount>;
322
323def gi_extract_cpol : GICustomOperandRenderer<"renderExtractCPol">,
324  GISDNodeXFormEquiv<extract_cpol>;
325
326def gi_extract_swz : GICustomOperandRenderer<"renderExtractSWZ">,
327  GISDNodeXFormEquiv<extract_swz>;
328
329def gi_set_glc : GICustomOperandRenderer<"renderSetGLC">,
330  GISDNodeXFormEquiv<set_glc>;
331
332def gi_frameindex_to_targetframeindex : GICustomOperandRenderer<"renderFrameIndex">,
333  GISDNodeXFormEquiv<frameindex_to_targetframeindex>;
334