1//===-- SIInstrInfo.td - SI Instruction Infos -------------*- tablegen -*--===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9def isWave32 : Predicate<"Subtarget->getWavefrontSize() == 32">,
10  AssemblerPredicate <(all_of FeatureWavefrontSize32)>;
11def isWave64 : Predicate<"Subtarget->getWavefrontSize() == 64">,
12  AssemblerPredicate <(all_of FeatureWavefrontSize64)>;
13
14class GCNPredicateControl : PredicateControl {
15  Predicate SIAssemblerPredicate = isGFX6GFX7;
16  Predicate VIAssemblerPredicate = isGFX8GFX9;
17}
18
19// Except for the NONE field, this must be kept in sync with the
20// SIEncodingFamily enum in SIInstrInfo.cpp and the columns of the
21// getMCOpcodeGen table.
22def SIEncodingFamily {
23  int NONE = -1;
24  int SI = 0;
25  int VI = 1;
26  int SDWA = 2;
27  int SDWA9 = 3;
28  int GFX80 = 4;
29  int GFX9 = 5;
30  int GFX10 = 6;
31  int SDWA10 = 7;
32  int GFX90A = 8;
33  int GFX940 = 9;
34  int GFX11 = 10;
35}
36
37//===----------------------------------------------------------------------===//
38// SI DAG Nodes
39//===----------------------------------------------------------------------===//
40
41def AMDGPUclamp : SDNode<"AMDGPUISD::CLAMP", SDTFPUnaryOp>;
42
43def SIsbuffer_load : SDNode<"AMDGPUISD::SBUFFER_LOAD",
44  SDTypeProfile<1, 3, [SDTCisVT<1, v4i32>, SDTCisVT<2, i32>, SDTCisVT<3, i32>]>,
45  [SDNPMayLoad, SDNPMemOperand]
46>;
47
48def SIds_ordered_count : SDNode<"AMDGPUISD::DS_ORDERED_COUNT",
49  SDTypeProfile<1, 2, [SDTCisVT<0, i32>, SDTCisVT<1, i32>, SDTCisVT<2, i16>]>,
50  [SDNPMayLoad, SDNPMayStore, SDNPMemOperand, SDNPHasChain, SDNPInGlue]
51>;
52
53def SIatomic_inc : SDNode<"AMDGPUISD::ATOMIC_INC", SDTAtomic2,
54  [SDNPMayLoad, SDNPMayStore, SDNPMemOperand, SDNPHasChain]
55>;
56
57def SIatomic_dec : SDNode<"AMDGPUISD::ATOMIC_DEC", SDTAtomic2,
58  [SDNPMayLoad, SDNPMayStore, SDNPMemOperand, SDNPHasChain]
59>;
60
61def SDTAtomic2_f32 : SDTypeProfile<1, 2, [
62  SDTCisSameAs<0,2>, SDTCisFP<0>, SDTCisPtrTy<1>
63]>;
64
65def SIatomic_fmin : SDNode<"AMDGPUISD::ATOMIC_LOAD_FMIN", SDTAtomic2_f32,
66  [SDNPMayLoad, SDNPMayStore, SDNPMemOperand, SDNPHasChain]
67>;
68
69def SIatomic_fmax : SDNode<"AMDGPUISD::ATOMIC_LOAD_FMAX", SDTAtomic2_f32,
70  [SDNPMayLoad, SDNPMayStore, SDNPMemOperand, SDNPHasChain]
71>;
72
73// load_d16_{lo|hi} ptr, tied_input
74def SIload_d16 : SDTypeProfile<1, 2, [
75  SDTCisPtrTy<1>,
76  SDTCisSameAs<0, 2>
77]>;
78
79
80def SDTtbuffer_load : SDTypeProfile<1, 8,
81  [                     // vdata
82   SDTCisVT<1, v4i32>,  // rsrc
83   SDTCisVT<2, i32>,    // vindex(VGPR)
84   SDTCisVT<3, i32>,    // voffset(VGPR)
85   SDTCisVT<4, i32>,    // soffset(SGPR)
86   SDTCisVT<5, i32>,    // offset(imm)
87   SDTCisVT<6, i32>,    // format(imm)
88   SDTCisVT<7, i32>,    // cachepolicy, swizzled buffer(imm)
89   SDTCisVT<8, i1>      // idxen(imm)
90  ]>;
91
92def SItbuffer_load :   SDNode<"AMDGPUISD::TBUFFER_LOAD_FORMAT", SDTtbuffer_load,
93                              [SDNPMayLoad, SDNPMemOperand, SDNPHasChain]>;
94def SItbuffer_load_d16 : SDNode<"AMDGPUISD::TBUFFER_LOAD_FORMAT_D16",
95                                SDTtbuffer_load,
96                                [SDNPMayLoad, SDNPMemOperand, SDNPHasChain]>;
97
98def SDTtbuffer_store : SDTypeProfile<0, 9,
99    [                     // vdata
100     SDTCisVT<1, v4i32>,  // rsrc
101     SDTCisVT<2, i32>,    // vindex(VGPR)
102     SDTCisVT<3, i32>,    // voffset(VGPR)
103     SDTCisVT<4, i32>,    // soffset(SGPR)
104     SDTCisVT<5, i32>,    // offset(imm)
105     SDTCisVT<6, i32>,    // format(imm)
106     SDTCisVT<7, i32>,    // cachepolicy, swizzled buffer(imm)
107     SDTCisVT<8, i1>      // idxen(imm)
108    ]>;
109
110def SItbuffer_store : SDNode<"AMDGPUISD::TBUFFER_STORE_FORMAT", SDTtbuffer_store,
111                             [SDNPMayStore, SDNPMemOperand, SDNPHasChain]>;
112def SItbuffer_store_d16 : SDNode<"AMDGPUISD::TBUFFER_STORE_FORMAT_D16",
113                                SDTtbuffer_store,
114                                [SDNPMayStore, SDNPMemOperand, SDNPHasChain]>;
115
116def SDTBufferLoad : SDTypeProfile<1, 7,
117    [                    // vdata
118     SDTCisVT<1, v4i32>, // rsrc
119     SDTCisVT<2, i32>,   // vindex(VGPR)
120     SDTCisVT<3, i32>,   // voffset(VGPR)
121     SDTCisVT<4, i32>,   // soffset(SGPR)
122     SDTCisVT<5, i32>,   // offset(imm)
123     SDTCisVT<6, i32>,   // cachepolicy, swizzled buffer(imm)
124     SDTCisVT<7, i1>]>;  // idxen(imm)
125
126def SIbuffer_load : SDNode <"AMDGPUISD::BUFFER_LOAD", SDTBufferLoad,
127                            [SDNPMemOperand, SDNPHasChain, SDNPMayLoad]>;
128def SIbuffer_load_ubyte : SDNode <"AMDGPUISD::BUFFER_LOAD_UBYTE", SDTBufferLoad,
129                            [SDNPMemOperand, SDNPHasChain, SDNPMayLoad]>;
130def SIbuffer_load_ushort : SDNode <"AMDGPUISD::BUFFER_LOAD_USHORT", SDTBufferLoad,
131                            [SDNPMemOperand, SDNPHasChain, SDNPMayLoad]>;
132def SIbuffer_load_byte : SDNode <"AMDGPUISD::BUFFER_LOAD_BYTE", SDTBufferLoad,
133                            [SDNPMemOperand, SDNPHasChain, SDNPMayLoad]>;
134def SIbuffer_load_short: SDNode <"AMDGPUISD::BUFFER_LOAD_SHORT", SDTBufferLoad,
135                            [SDNPMemOperand, SDNPHasChain, SDNPMayLoad]>;
136def SIbuffer_load_format : SDNode <"AMDGPUISD::BUFFER_LOAD_FORMAT", SDTBufferLoad,
137                            [SDNPMemOperand, SDNPHasChain, SDNPMayLoad]>;
138def SIbuffer_load_format_d16 : SDNode <"AMDGPUISD::BUFFER_LOAD_FORMAT_D16",
139                                SDTBufferLoad,
140                                [SDNPMemOperand, SDNPHasChain, SDNPMayLoad]>;
141
142def SDTBufferStore : SDTypeProfile<0, 8,
143    [                    // vdata
144     SDTCisVT<1, v4i32>, // rsrc
145     SDTCisVT<2, i32>,   // vindex(VGPR)
146     SDTCisVT<3, i32>,   // voffset(VGPR)
147     SDTCisVT<4, i32>,   // soffset(SGPR)
148     SDTCisVT<5, i32>,   // offset(imm)
149     SDTCisVT<6, i32>,   // cachepolicy, swizzled buffer(imm)
150     SDTCisVT<7, i1>]>;  // idxen(imm)
151
152def SIbuffer_store : SDNode <"AMDGPUISD::BUFFER_STORE", SDTBufferStore,
153                             [SDNPMayStore, SDNPMemOperand, SDNPHasChain]>;
154def SIbuffer_store_byte: SDNode <"AMDGPUISD::BUFFER_STORE_BYTE",
155                         SDTBufferStore,
156                         [SDNPMayStore, SDNPMemOperand, SDNPHasChain]>;
157def SIbuffer_store_short : SDNode <"AMDGPUISD::BUFFER_STORE_SHORT",
158                           SDTBufferStore,
159                           [SDNPMayStore, SDNPMemOperand, SDNPHasChain]>;
160def SIbuffer_store_format : SDNode <"AMDGPUISD::BUFFER_STORE_FORMAT",
161                            SDTBufferStore,
162                            [SDNPMayStore, SDNPMemOperand, SDNPHasChain]>;
163def SIbuffer_store_format_d16 : SDNode <"AMDGPUISD::BUFFER_STORE_FORMAT_D16",
164                            SDTBufferStore,
165                            [SDNPMayStore, SDNPMemOperand, SDNPHasChain]>;
166
167class SDBufferAtomic<string opcode> : SDNode <opcode,
168  SDTypeProfile<1, 8,
169       [SDTCisVT<2, v4i32>, // rsrc
170       SDTCisVT<3, i32>,   // vindex(VGPR)
171       SDTCisVT<4, i32>,   // voffset(VGPR)
172       SDTCisVT<5, i32>,   // soffset(SGPR)
173       SDTCisVT<6, i32>,   // offset(imm)
174       SDTCisVT<7, i32>,   // cachepolicy(imm)
175       SDTCisVT<8, i1>]>,  // idxen(imm)
176  [SDNPMemOperand, SDNPHasChain, SDNPMayLoad, SDNPMayStore]
177>;
178
179def SIbuffer_atomic_swap : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_SWAP">;
180def SIbuffer_atomic_add : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_ADD">;
181def SIbuffer_atomic_sub : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_SUB">;
182def SIbuffer_atomic_smin : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_SMIN">;
183def SIbuffer_atomic_umin : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_UMIN">;
184def SIbuffer_atomic_smax : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_SMAX">;
185def SIbuffer_atomic_umax : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_UMAX">;
186def SIbuffer_atomic_and : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_AND">;
187def SIbuffer_atomic_or : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_OR">;
188def SIbuffer_atomic_xor : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_XOR">;
189def SIbuffer_atomic_inc : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_INC">;
190def SIbuffer_atomic_dec : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_DEC">;
191def SIbuffer_atomic_csub : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_CSUB">;
192def SIbuffer_atomic_fadd : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_FADD">;
193def SIbuffer_atomic_fmin : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_FMIN">;
194def SIbuffer_atomic_fmax : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_FMAX">;
195
196multiclass SDBufferAtomicRetNoRet {
197  def "_ret" : PatFrag<
198    (ops node:$vdata_in, node:$rsrc, node:$vindex, node:$voffset, node:$soffset,
199      node:$offset, node:$cachepolicy, node:$idxen),
200    (!cast<SDNode>(NAME) node:$vdata_in, node:$rsrc, node:$vindex,
201      node:$voffset, node:$soffset, node:$offset, node:$cachepolicy,
202      node:$idxen)> {
203    let PredicateCode = [{ return !(SDValue(N, 0).use_empty()); }];
204    let GISelPredicateCode = [{ return true; }];
205  }
206
207  def "_noret" : PatFrag<
208    (ops node:$vdata_in, node:$rsrc, node:$vindex, node:$voffset, node:$soffset,
209      node:$offset, node:$cachepolicy, node:$idxen),
210    (!cast<SDNode>(NAME) node:$vdata_in, node:$rsrc, node:$vindex,
211      node:$voffset, node:$soffset, node:$offset, node:$cachepolicy,
212      node:$idxen)> {
213    let PredicateCode = [{ return SDValue(N, 0).use_empty(); }];
214    let GISelPredicateCode = [{ return false; }];
215  }
216}
217
218defm SIbuffer_atomic_swap : SDBufferAtomicRetNoRet;
219defm SIbuffer_atomic_add : SDBufferAtomicRetNoRet;
220defm SIbuffer_atomic_sub : SDBufferAtomicRetNoRet;
221defm SIbuffer_atomic_smin : SDBufferAtomicRetNoRet;
222defm SIbuffer_atomic_umin : SDBufferAtomicRetNoRet;
223defm SIbuffer_atomic_smax : SDBufferAtomicRetNoRet;
224defm SIbuffer_atomic_umax : SDBufferAtomicRetNoRet;
225defm SIbuffer_atomic_and : SDBufferAtomicRetNoRet;
226defm SIbuffer_atomic_or : SDBufferAtomicRetNoRet;
227defm SIbuffer_atomic_xor : SDBufferAtomicRetNoRet;
228defm SIbuffer_atomic_inc : SDBufferAtomicRetNoRet;
229defm SIbuffer_atomic_dec : SDBufferAtomicRetNoRet;
230defm SIbuffer_atomic_fadd : SDBufferAtomicRetNoRet;
231defm SIbuffer_atomic_fmin : SDBufferAtomicRetNoRet;
232defm SIbuffer_atomic_fmax : SDBufferAtomicRetNoRet;
233
234def SIbuffer_atomic_cmpswap : SDNode <"AMDGPUISD::BUFFER_ATOMIC_CMPSWAP",
235  SDTypeProfile<1, 9,
236    [SDTCisVT<0, i32>,   // dst
237     SDTCisVT<1, i32>,   // src
238     SDTCisVT<2, i32>,   // cmp
239     SDTCisVT<3, v4i32>, // rsrc
240     SDTCisVT<4, i32>,   // vindex(VGPR)
241     SDTCisVT<5, i32>,   // voffset(VGPR)
242     SDTCisVT<6, i32>,   // soffset(SGPR)
243     SDTCisVT<7, i32>,   // offset(imm)
244     SDTCisVT<8, i32>,   // cachepolicy(imm)
245     SDTCisVT<9, i1>]>,  // idxen(imm)
246  [SDNPMemOperand, SDNPHasChain, SDNPMayLoad, SDNPMayStore]
247>;
248
249def SIbuffer_atomic_cmpswap_ret : PatFrag<
250  (ops node:$src, node:$cmp, node:$rsrc, node:$vindex, node:$voffset,
251    node:$soffset, node:$offset, node:$cachepolicy, node:$idxen),
252  (SIbuffer_atomic_cmpswap node:$src, node:$cmp, node:$rsrc, node:$vindex,
253    node:$voffset, node:$soffset, node:$offset, node:$cachepolicy,
254    node:$idxen)> {
255  let PredicateCode = [{ return !(SDValue(N, 0).use_empty()); }];
256  let GISelPredicateCode = [{ return true; }];
257}
258
259def SIbuffer_atomic_cmpswap_noret : PatFrag<
260  (ops node:$src, node:$cmp, node:$rsrc, node:$vindex, node:$voffset,
261    node:$soffset, node:$offset, node:$cachepolicy, node:$idxen),
262  (SIbuffer_atomic_cmpswap node:$src, node:$cmp, node:$rsrc, node:$vindex,
263    node:$voffset, node:$soffset, node:$offset, node:$cachepolicy,
264    node:$idxen)> {
265  let PredicateCode = [{ return SDValue(N, 0).use_empty(); }];
266  let GISelPredicateCode = [{ return false; }];
267}
268
269class SDGlobalAtomicNoRtn<string opcode, ValueType ty> : SDNode <opcode,
270  SDTypeProfile<0, 2,
271      [SDTCisPtrTy<0>,     // vaddr
272       SDTCisVT<1, ty>]>,  // vdata
273  [SDNPMemOperand, SDNPHasChain, SDNPMayLoad, SDNPMayStore]
274>;
275
276def SIpc_add_rel_offset : SDNode<"AMDGPUISD::PC_ADD_REL_OFFSET",
277  SDTypeProfile<1, 2, [SDTCisVT<0, iPTR>, SDTCisSameAs<0,1>, SDTCisSameAs<0,2>]>
278>;
279
280def SIlds : SDNode<"AMDGPUISD::LDS",
281  SDTypeProfile<1, 1, [SDTCisVT<0, iPTR>, SDTCisSameAs<0,1>]>
282>;
283
284def SIload_d16_lo : SDNode<"AMDGPUISD::LOAD_D16_LO",
285  SIload_d16,
286  [SDNPMayLoad, SDNPMemOperand, SDNPHasChain]
287>;
288
289def SIload_d16_lo_u8 : SDNode<"AMDGPUISD::LOAD_D16_LO_U8",
290  SIload_d16,
291  [SDNPMayLoad, SDNPMemOperand, SDNPHasChain]
292>;
293
294def SIload_d16_lo_i8 : SDNode<"AMDGPUISD::LOAD_D16_LO_I8",
295  SIload_d16,
296  [SDNPMayLoad, SDNPMemOperand, SDNPHasChain]
297>;
298
299def SIload_d16_hi : SDNode<"AMDGPUISD::LOAD_D16_HI",
300  SIload_d16,
301  [SDNPMayLoad, SDNPMemOperand, SDNPHasChain]
302>;
303
304def SIload_d16_hi_u8 : SDNode<"AMDGPUISD::LOAD_D16_HI_U8",
305  SIload_d16,
306  [SDNPMayLoad, SDNPMemOperand, SDNPHasChain]
307>;
308
309def SIload_d16_hi_i8 : SDNode<"AMDGPUISD::LOAD_D16_HI_I8",
310  SIload_d16,
311  [SDNPMayLoad, SDNPMemOperand, SDNPHasChain]
312>;
313
314def SIdenorm_mode : SDNode<"AMDGPUISD::DENORM_MODE",
315  SDTypeProfile<0 ,1, [SDTCisInt<0>]>,
316  [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]
317>;
318
319def SIfptrunc_round_upward : SDNode<"AMDGPUISD::FPTRUNC_ROUND_UPWARD",
320  SDTFPRoundOp
321>;
322
323def SIfptrunc_round_downward : SDNode<"AMDGPUISD::FPTRUNC_ROUND_DOWNWARD",
324  SDTFPRoundOp
325>;
326
327//===----------------------------------------------------------------------===//
328// ValueType helpers
329//===----------------------------------------------------------------------===//
330
331// Returns 1 if the source arguments have modifiers, 0 if they do not.
332// XXX - do f16 instructions?
333class isFloatType<ValueType SrcVT> {
334  bit ret = !or(!eq(SrcVT.Value, f16.Value),
335                !eq(SrcVT.Value, f32.Value),
336                !eq(SrcVT.Value, f64.Value),
337                !eq(SrcVT.Value, v2f16.Value),
338                !eq(SrcVT.Value, v4f16.Value),
339                !eq(SrcVT.Value, v2f32.Value),
340                !eq(SrcVT.Value, v2f64.Value),
341                !eq(SrcVT.Value, v4f64.Value));
342}
343
344class isIntType<ValueType SrcVT> {
345  bit ret = !or(!eq(SrcVT.Value, i16.Value),
346                !eq(SrcVT.Value, i32.Value),
347                !eq(SrcVT.Value, i64.Value),
348                !eq(SrcVT.Value, v2i32.Value));
349}
350
351class isPackedType<ValueType SrcVT> {
352  bit ret = !or(!eq(SrcVT.Value, v2i16.Value),
353                !eq(SrcVT.Value, v2f16.Value),
354                !eq(SrcVT.Value, v4f16.Value),
355                !eq(SrcVT.Value, v2f32.Value));
356}
357
358
359//===----------------------------------------------------------------------===//
360// PatFrags for global memory operations
361//===----------------------------------------------------------------------===//
362
363defm atomic_inc : binary_atomic_op_all_as<SIatomic_inc>;
364defm atomic_dec : binary_atomic_op_all_as<SIatomic_dec>;
365defm atomic_load_fmin : binary_atomic_op_all_as<SIatomic_fmin, 0>;
366defm atomic_load_fmax : binary_atomic_op_all_as<SIatomic_fmax, 0>;
367
368//===----------------------------------------------------------------------===//
369// SDNodes PatFrags for loads/stores with a glue input.
370// This is for SDNodes and PatFrag for local loads and stores to
371// enable s_mov_b32 m0, -1 to be glued to the memory instructions.
372//
373// These mirror the regular load/store PatFrags and rely on special
374// processing during Select() to add the glued copy.
375//
376//===----------------------------------------------------------------------===//
377
378def AMDGPUld_glue : SDNode <"ISD::LOAD", SDTLoad,
379  [SDNPHasChain, SDNPMayLoad, SDNPMemOperand, SDNPInGlue]
380>;
381
382def AMDGPUatomic_ld_glue : SDNode <"ISD::ATOMIC_LOAD", SDTAtomicLoad,
383  [SDNPHasChain, SDNPMayLoad, SDNPMemOperand, SDNPInGlue]
384>;
385
386def unindexedload_glue : PatFrag <(ops node:$ptr), (AMDGPUld_glue node:$ptr)> {
387  let IsLoad = 1;
388  let IsUnindexed = 1;
389}
390
391def load_glue : PatFrag <(ops node:$ptr), (unindexedload_glue node:$ptr)> {
392  let IsLoad = 1;
393  let IsNonExtLoad = 1;
394}
395
396def atomic_load_8_glue : PatFrag<(ops node:$ptr),
397  (AMDGPUatomic_ld_glue node:$ptr)> {
398  let IsAtomic = 1;
399  let MemoryVT = i8;
400}
401
402def atomic_load_16_glue : PatFrag<(ops node:$ptr),
403  (AMDGPUatomic_ld_glue node:$ptr)> {
404  let IsAtomic = 1;
405  let MemoryVT = i16;
406}
407
408def atomic_load_32_glue : PatFrag<(ops node:$ptr),
409  (AMDGPUatomic_ld_glue node:$ptr)> {
410  let IsAtomic = 1;
411  let MemoryVT = i32;
412}
413
414def atomic_load_64_glue : PatFrag<(ops node:$ptr),
415  (AMDGPUatomic_ld_glue node:$ptr)> {
416  let IsAtomic = 1;
417  let MemoryVT = i64;
418}
419
420def extload_glue : PatFrag<(ops node:$ptr), (unindexedload_glue node:$ptr)> {
421  let IsLoad = 1;
422  let IsAnyExtLoad = 1;
423}
424
425def sextload_glue : PatFrag<(ops node:$ptr), (unindexedload_glue node:$ptr)> {
426  let IsLoad = 1;
427  let IsSignExtLoad = 1;
428}
429
430def zextload_glue : PatFrag<(ops node:$ptr), (unindexedload_glue node:$ptr)> {
431  let IsLoad = 1;
432  let IsZeroExtLoad = 1;
433}
434
435def extloadi8_glue : PatFrag<(ops node:$ptr), (extload_glue node:$ptr)> {
436  let IsLoad = 1;
437  let MemoryVT = i8;
438}
439
440def zextloadi8_glue : PatFrag<(ops node:$ptr), (zextload_glue node:$ptr)> {
441  let IsLoad = 1;
442  let MemoryVT = i8;
443}
444
445def extloadi16_glue : PatFrag<(ops node:$ptr), (extload_glue node:$ptr)> {
446  let IsLoad = 1;
447  let MemoryVT = i16;
448}
449
450def zextloadi16_glue : PatFrag<(ops node:$ptr), (zextload_glue node:$ptr)> {
451  let IsLoad = 1;
452  let MemoryVT = i16;
453}
454
455def sextloadi8_glue : PatFrag<(ops node:$ptr), (sextload_glue node:$ptr)> {
456  let IsLoad = 1;
457  let MemoryVT = i8;
458}
459
460def sextloadi16_glue : PatFrag<(ops node:$ptr), (sextload_glue node:$ptr)> {
461  let IsLoad = 1;
462  let MemoryVT = i16;
463}
464
465
466let IsLoad = 1, AddressSpaces = LoadAddress_local.AddrSpaces in {
467def load_local_m0 : PatFrag<(ops node:$ptr), (load_glue node:$ptr)> {
468  let IsNonExtLoad = 1;
469}
470
471def extloadi8_local_m0 : PatFrag<(ops node:$ptr), (extloadi8_glue node:$ptr)>;
472def sextloadi8_local_m0 : PatFrag<(ops node:$ptr), (sextloadi8_glue node:$ptr)>;
473def zextloadi8_local_m0 : PatFrag<(ops node:$ptr), (zextloadi8_glue node:$ptr)>;
474
475def extloadi16_local_m0 : PatFrag<(ops node:$ptr), (extloadi16_glue node:$ptr)>;
476def sextloadi16_local_m0 : PatFrag<(ops node:$ptr), (sextloadi16_glue node:$ptr)>;
477def zextloadi16_local_m0 : PatFrag<(ops node:$ptr), (zextloadi16_glue node:$ptr)>;
478} // End IsLoad = 1, , AddressSpaces = LoadAddress_local.AddrSpaces
479
480def load_align8_local_m0 : PatFrag<(ops node:$ptr),
481                                   (load_local_m0 node:$ptr)> {
482  let IsLoad = 1;
483  int MinAlignment = 8;
484}
485
486def load_align16_local_m0 : PatFrag<(ops node:$ptr),
487                                   (load_local_m0 node:$ptr)> {
488  let IsLoad = 1;
489  int MinAlignment = 16;
490}
491
492let IsAtomic = 1, AddressSpaces = LoadAddress_local.AddrSpaces in {
493def atomic_load_8_local_m0 : PatFrag<(ops node:$ptr),
494                                      (atomic_load_8_glue node:$ptr)>;
495def atomic_load_16_local_m0 : PatFrag<(ops node:$ptr),
496                                      (atomic_load_16_glue node:$ptr)>;
497def atomic_load_32_local_m0 : PatFrag<(ops node:$ptr),
498                                      (atomic_load_32_glue node:$ptr)>;
499def atomic_load_64_local_m0 : PatFrag<(ops node:$ptr),
500                                       (atomic_load_64_glue node:$ptr)>;
501} // End let AddressSpaces = LoadAddress_local.AddrSpaces
502
503
504def AMDGPUst_glue : SDNode <"ISD::STORE", SDTStore,
505  [SDNPHasChain, SDNPMayStore, SDNPMemOperand, SDNPInGlue]
506>;
507
508def AMDGPUatomic_st_glue : SDNode <"ISD::ATOMIC_STORE", SDTAtomicStore,
509  [SDNPHasChain, SDNPMayStore, SDNPMemOperand, SDNPInGlue]
510>;
511
512def unindexedstore_glue : PatFrag<(ops node:$val, node:$ptr),
513                                   (AMDGPUst_glue node:$val, node:$ptr)> {
514  let IsStore = 1;
515  let IsUnindexed = 1;
516}
517
518def store_glue : PatFrag<(ops node:$val, node:$ptr),
519                         (unindexedstore_glue node:$val, node:$ptr)> {
520  let IsStore = 1;
521  let IsTruncStore = 0;
522}
523
524def truncstore_glue : PatFrag<(ops node:$val, node:$ptr),
525  (unindexedstore_glue node:$val, node:$ptr)> {
526  let IsStore = 1;
527  let IsTruncStore = 1;
528}
529
530def truncstorei8_glue : PatFrag<(ops node:$val, node:$ptr),
531                           (truncstore_glue node:$val, node:$ptr)> {
532  let IsStore = 1;
533  let MemoryVT = i8;
534  let IsTruncStore = 1;
535}
536
537def truncstorei16_glue : PatFrag<(ops node:$val, node:$ptr),
538                           (truncstore_glue node:$val, node:$ptr)> {
539  let IsStore = 1;
540  let MemoryVT = i16;
541  let IsTruncStore = 1;
542}
543
544let IsStore = 1, AddressSpaces = StoreAddress_local.AddrSpaces in {
545def store_local_m0 : PatFrag<(ops node:$val, node:$ptr),
546                             (store_glue node:$val, node:$ptr)>;
547def truncstorei8_local_m0 : PatFrag<(ops node:$val, node:$ptr),
548                                    (truncstorei8_glue node:$val, node:$ptr)>;
549def truncstorei16_local_m0 : PatFrag<(ops node:$val, node:$ptr),
550                                    (truncstorei16_glue node:$val, node:$ptr)>;
551}
552
553def store_align8_local_m0 : PatFrag <(ops node:$value, node:$ptr),
554                                     (store_local_m0 node:$value, node:$ptr)>,
555                            Aligned<8> {
556  let IsStore = 1;
557}
558
559def store_align16_local_m0 : PatFrag <(ops node:$value, node:$ptr),
560                                     (store_local_m0 node:$value, node:$ptr)>,
561                            Aligned<16> {
562  let IsStore = 1;
563}
564
565let PredicateCode = [{return cast<MemSDNode>(N)->getAlignment() < 4;}],
566    GISelPredicateCode = [{return (*MI.memoperands_begin())->getAlign() < 4;}],
567    AddressSpaces = [ AddrSpaces.Local ] in {
568def load_align_less_than_4_local : PatFrag<(ops node:$ptr),
569                                           (load_local node:$ptr)> {
570  let IsLoad = 1;
571  let IsNonExtLoad = 1;
572}
573
574def load_align_less_than_4_local_m0 : PatFrag<(ops node:$ptr),
575                                              (load_local_m0 node:$ptr)> {
576  let IsLoad = 1;
577  let IsNonExtLoad = 1;
578}
579
580def store_align_less_than_4_local : PatFrag <(ops node:$value, node:$ptr),
581                                             (store_local node:$value, node:$ptr)> {
582  let IsStore = 1;
583  let IsTruncStore = 0;
584}
585
586def store_align_less_than_4_local_m0 : PatFrag <(ops node:$value, node:$ptr),
587                                                (store_local_m0 node:$value, node:$ptr)> {
588  let IsStore = 1;
589  let IsTruncStore = 0;
590}
591}
592
593def atomic_store_8_glue : PatFrag <
594  (ops node:$ptr, node:$value),
595  (AMDGPUatomic_st_glue node:$ptr, node:$value)> {
596  let IsAtomic = 1;
597  let MemoryVT = i8;
598}
599
600def atomic_store_16_glue : PatFrag <
601  (ops node:$ptr, node:$value),
602  (AMDGPUatomic_st_glue node:$ptr, node:$value)> {
603  let IsAtomic = 1;
604  let MemoryVT = i16;
605}
606
607def atomic_store_32_glue : PatFrag <
608  (ops node:$ptr, node:$value),
609  (AMDGPUatomic_st_glue node:$ptr, node:$value)> {
610  let IsAtomic = 1;
611  let MemoryVT = i32;
612}
613
614def atomic_store_64_glue : PatFrag <
615  (ops node:$ptr, node:$value),
616  (AMDGPUatomic_st_glue node:$ptr, node:$value)> {
617  let IsAtomic = 1;
618  let MemoryVT = i64;
619}
620
621let IsAtomic = 1, AddressSpaces = StoreAddress_local.AddrSpaces in {
622def atomic_store_8_local_m0 : PatFrag<(ops node:$ptr, node:$val),
623                                       (atomic_store_8_glue node:$ptr, node:$val)>;
624def atomic_store_16_local_m0 : PatFrag<(ops node:$ptr, node:$val),
625                                       (atomic_store_16_glue node:$ptr, node:$val)>;
626def atomic_store_32_local_m0 : PatFrag<(ops node:$ptr, node:$val),
627                                       (atomic_store_32_glue node:$ptr, node:$val)>;
628def atomic_store_64_local_m0 : PatFrag<(ops node:$ptr, node:$val),
629                                       (atomic_store_64_glue node:$ptr, node:$val)>;
630} // End let IsAtomic = 1, AddressSpaces = StoreAddress_local.AddrSpaces
631
632
633def si_setcc_uniform : PatFrag <
634  (ops node:$lhs, node:$rhs, node:$cond),
635  (setcc node:$lhs, node:$rhs, node:$cond), [{
636  return !N->isDivergent();
637}]>;
638
639//===----------------------------------------------------------------------===//
640// SDNodes PatFrags for a16 loads and stores with 3 components.
641// v3f16/v3i16 is widened to v4f16/v4i16, so we need to match on the memory
642// load/store size.
643//===----------------------------------------------------------------------===//
644
645class mubuf_intrinsic_load<SDPatternOperator name, ValueType vt> : PatFrag <
646  (ops node:$rsrc, node:$vindex, node:$voffset, node:$soffset, node:$offset,
647            node:$auxiliary, node:$idxen),
648  (name node:$rsrc, node:$vindex, node:$voffset, node:$soffset, node:$offset,
649            node:$auxiliary, node:$idxen)> {
650  let IsLoad = 1;
651  let MemoryVT = vt;
652}
653
654class mubuf_intrinsic_store<SDPatternOperator name, ValueType vt> : PatFrag <
655  (ops node:$vdata, node:$rsrc, node:$vindex, node:$voffset, node:$soffset, node:$offset,
656            node:$auxiliary, node:$idxen),
657  (name node:$vdata, node:$rsrc, node:$vindex, node:$voffset, node:$soffset, node:$offset,
658            node:$auxiliary, node:$idxen)> {
659  let IsStore = 1;
660  let MemoryVT = vt;
661}
662
663class mtbuf_intrinsic_load<SDPatternOperator name, ValueType vt> : PatFrag <
664  (ops node:$rsrc, node:$vindex, node:$voffset, node:$soffset, node:$offset,
665            node:$format, node:$auxiliary, node:$idxen),
666  (name node:$rsrc, node:$vindex, node:$voffset, node:$soffset, node:$offset,
667            node:$format, node:$auxiliary, node:$idxen)> {
668  let IsLoad = 1;
669  let MemoryVT = vt;
670}
671
672class mtbuf_intrinsic_store<SDPatternOperator name, ValueType vt> : PatFrag <
673  (ops node:$vdata, node:$rsrc, node:$vindex, node:$voffset, node:$soffset, node:$offset,
674            node:$format, node:$auxiliary, node:$idxen),
675  (name node:$vdata, node:$rsrc, node:$vindex, node:$voffset, node:$soffset, node:$offset,
676            node:$format, node:$auxiliary, node:$idxen)> {
677  let IsStore = 1;
678  let MemoryVT = vt;
679}
680
681//===----------------------------------------------------------------------===//
682// SDNodes PatFrags for d16 loads
683//===----------------------------------------------------------------------===//
684
685class LoadD16Frag <SDPatternOperator op> : PatFrag<
686  (ops node:$ptr, node:$tied_in),
687  (op node:$ptr, node:$tied_in)> {
688  let IsLoad = 1;
689}
690
691foreach as = [ "global", "flat", "constant", "local", "private", "region" ] in {
692let AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in {
693
694def load_d16_hi_#as : LoadD16Frag <SIload_d16_hi>;
695
696def az_extloadi8_d16_hi_#as : LoadD16Frag <SIload_d16_hi_u8> {
697  let MemoryVT = i8;
698}
699
700def sextloadi8_d16_hi_#as : LoadD16Frag <SIload_d16_hi_i8> {
701  let MemoryVT = i8;
702}
703
704def load_d16_lo_#as : LoadD16Frag <SIload_d16_lo>;
705
706def az_extloadi8_d16_lo_#as : LoadD16Frag <SIload_d16_lo_u8> {
707  let MemoryVT = i8;
708}
709
710def sextloadi8_d16_lo_#as : LoadD16Frag <SIload_d16_lo_i8> {
711  let MemoryVT = i8;
712}
713
714} // End let AddressSpaces = ...
715} // End foreach AddrSpace
716
717def lshr_rev : PatFrag <
718  (ops node:$src1, node:$src0),
719  (srl $src0, $src1)
720>;
721
722def ashr_rev : PatFrag <
723  (ops node:$src1, node:$src0),
724  (sra $src0, $src1)
725>;
726
727def lshl_rev : PatFrag <
728  (ops node:$src1, node:$src0),
729  (shl $src0, $src1)
730>;
731
732def add_ctpop : PatFrag <
733  (ops node:$src0, node:$src1),
734  (add (ctpop $src0), $src1)
735>;
736
737def xnor : PatFrag <
738  (ops node:$src0, node:$src1),
739  (not (xor $src0, $src1))
740>;
741
742foreach I = 1-4 in {
743def shl#I#_add : PatFrag <
744  (ops node:$src0, node:$src1),
745  (add (shl_oneuse $src0, (i32 I)), $src1)> {
746  // FIXME: Poor substitute for disabling pattern in SelectionDAG
747  let PredicateCode = [{return false;}];
748  let GISelPredicateCode = [{return true;}];
749}
750}
751
752multiclass SIAtomicM0Glue2 <string op_name, bit is_amdgpu = 0,
753                            SDTypeProfile tc = SDTAtomic2,
754                            bit IsInt = 1> {
755
756  def _glue : SDNode <
757    !if(is_amdgpu, "AMDGPUISD", "ISD")#"::ATOMIC_"#op_name, tc,
758    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand, SDNPInGlue]
759  >;
760
761  let AddressSpaces = StoreAddress_local.AddrSpaces in {
762    defm _local_m0 : binary_atomic_op <!cast<SDNode>(NAME#"_glue"), IsInt>;
763    defm _local_m0 : ret_noret_binary_atomic_op <!cast<SDNode>(NAME#"_glue"),
764                                                 IsInt>;
765  }
766
767  let AddressSpaces = StoreAddress_region.AddrSpaces in {
768    defm _region_m0 : binary_atomic_op <!cast<SDNode>(NAME#"_glue"), IsInt>;
769    defm _region_m0 : ret_noret_binary_atomic_op <!cast<SDNode>(NAME#"_glue"),
770                                                  IsInt>;
771  }
772}
773
774defm atomic_load_add : SIAtomicM0Glue2 <"LOAD_ADD">;
775defm atomic_load_sub : SIAtomicM0Glue2 <"LOAD_SUB">;
776defm atomic_inc : SIAtomicM0Glue2 <"INC", 1>;
777defm atomic_dec : SIAtomicM0Glue2 <"DEC", 1>;
778defm atomic_load_and : SIAtomicM0Glue2 <"LOAD_AND">;
779defm atomic_load_min : SIAtomicM0Glue2 <"LOAD_MIN">;
780defm atomic_load_max : SIAtomicM0Glue2 <"LOAD_MAX">;
781defm atomic_load_or : SIAtomicM0Glue2 <"LOAD_OR">;
782defm atomic_load_xor : SIAtomicM0Glue2 <"LOAD_XOR">;
783defm atomic_load_umin : SIAtomicM0Glue2 <"LOAD_UMIN">;
784defm atomic_load_umax : SIAtomicM0Glue2 <"LOAD_UMAX">;
785defm atomic_swap : SIAtomicM0Glue2 <"SWAP">;
786defm atomic_load_fadd : SIAtomicM0Glue2 <"LOAD_FADD", 0, SDTAtomic2_f32, 0>;
787defm atomic_load_fmin : SIAtomicM0Glue2 <"LOAD_FMIN", 1, SDTAtomic2_f32, 0>;
788defm atomic_load_fmax : SIAtomicM0Glue2 <"LOAD_FMAX", 1, SDTAtomic2_f32, 0>;
789
790def as_i1timm : SDNodeXForm<timm, [{
791  return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i1);
792}]>;
793
794def as_i8imm : SDNodeXForm<imm, [{
795  return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i8);
796}]>;
797
798def as_i8timm : SDNodeXForm<timm, [{
799  return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i16);
800}]>;
801
802def as_i16imm : SDNodeXForm<imm, [{
803  return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i16);
804}]>;
805
806def as_i16timm : SDNodeXForm<timm, [{
807  return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i16);
808}]>;
809
810def as_i32imm: SDNodeXForm<imm, [{
811  return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i32);
812}]>;
813
814def as_i32timm: SDNodeXForm<timm, [{
815  return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i32);
816}]>;
817
818def as_i64imm: SDNodeXForm<imm, [{
819  return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i64);
820}]>;
821
822def cond_as_i32imm: SDNodeXForm<cond, [{
823  return CurDAG->getTargetConstant(N->get(), SDLoc(N), MVT::i32);
824}]>;
825
826// Copied from the AArch64 backend:
827def bitcast_fpimm_to_i32 : SDNodeXForm<fpimm, [{
828return CurDAG->getTargetConstant(
829  N->getValueAPF().bitcastToAPInt().getZExtValue(), SDLoc(N), MVT::i32);
830}]>;
831
832def frameindex_to_targetframeindex : SDNodeXForm<frameindex, [{
833  auto FI = cast<FrameIndexSDNode>(N);
834  return CurDAG->getTargetFrameIndex(FI->getIndex(), MVT::i32);
835}]>;
836
837// Copied from the AArch64 backend:
838def bitcast_fpimm_to_i64 : SDNodeXForm<fpimm, [{
839return CurDAG->getTargetConstant(
840  N->getValueAPF().bitcastToAPInt().getZExtValue(), SDLoc(N), MVT::i64);
841}]>;
842
843class bitextract_imm<int bitnum> : SDNodeXForm<imm, [{
844  uint64_t Imm = N->getZExtValue();
845  unsigned Bit = (Imm >> }] # bitnum # [{ ) & 1;
846  return CurDAG->getTargetConstant(Bit, SDLoc(N), MVT::i1);
847}]>;
848
849def SIMM16bit : ImmLeaf <i32,
850  [{return isInt<16>(Imm);}]
851>;
852
853def UIMM16bit : ImmLeaf <i32,
854  [{return isUInt<16>(Imm);}]
855>;
856
857def i64imm_32bit : ImmLeaf<i64, [{
858  return (Imm & 0xffffffffULL) == static_cast<uint64_t>(Imm);
859}]>;
860
861def InlineImm16 : ImmLeaf<i16, [{
862  return isInlineImmediate16(Imm);
863}]>;
864
865def InlineImm32 : ImmLeaf<i32, [{
866  return isInlineImmediate32(Imm);
867}]>;
868
869def InlineImm64 : ImmLeaf<i64, [{
870  return isInlineImmediate64(Imm);
871}]>;
872
873def InlineImmFP32 : FPImmLeaf<f32, [{
874  return isInlineImmediate(Imm);
875}]>;
876
877def InlineImmFP64 : FPImmLeaf<f64, [{
878  return isInlineImmediate(Imm);
879}]>;
880
881
882class VGPRImm <dag frag> : PatLeaf<frag, [{
883  return isVGPRImm(N);
884}]>;
885
886def NegateImm : SDNodeXForm<imm, [{
887  return CurDAG->getConstant(-N->getSExtValue(), SDLoc(N), MVT::i32);
888}]>;
889
890// TODO: When FP inline imm values work?
891def NegSubInlineConst32 : ImmLeaf<i32, [{
892  return Imm < -16 && Imm >= -64;
893}], NegateImm>;
894
895def NegSubInlineIntConst16 : ImmLeaf<i16, [{
896  return Imm < -16 && Imm >= -64;
897}], NegateImm>;
898
899def ShiftAmt32Imm : ImmLeaf <i32, [{
900  return Imm < 32;
901}]>;
902
903def getNegV2I16Imm : SDNodeXForm<build_vector, [{
904  return SDValue(packNegConstantV2I16(N, *CurDAG), 0);
905}]>;
906
907def NegSubInlineConstV216 : PatLeaf<(build_vector), [{
908  assert(N->getNumOperands() == 2);
909  assert(N->getOperand(0).getValueType().getSizeInBits() == 16);
910  SDValue Src0 = N->getOperand(0);
911  SDValue Src1 = N->getOperand(1);
912  if (Src0 == Src1)
913    return isNegInlineImmediate(Src0.getNode());
914
915  return (isNullConstantOrUndef(Src0) && isNegInlineImmediate(Src1.getNode())) ||
916         (isNullConstantOrUndef(Src1) && isNegInlineImmediate(Src0.getNode()));
917}], getNegV2I16Imm>;
918
919
920def fp16_zeros_high_16bits : PatLeaf<(f16 VGPR_32:$src), [{
921  return fp16SrcZerosHighBits(N->getOpcode());
922}]>;
923
924
925//===----------------------------------------------------------------------===//
926// MUBUF/SMEM Patterns
927//===----------------------------------------------------------------------===//
928
929def extract_cpol : SDNodeXForm<timm, [{
930  return CurDAG->getTargetConstant(N->getZExtValue() & AMDGPU::CPol::ALL, SDLoc(N), MVT::i8);
931}]>;
932
933def extract_swz : SDNodeXForm<timm, [{
934  return CurDAG->getTargetConstant((N->getZExtValue() >> 3) & 1, SDLoc(N), MVT::i8);
935}]>;
936
937def set_glc : SDNodeXForm<timm, [{
938  return CurDAG->getTargetConstant(N->getZExtValue() | AMDGPU::CPol::GLC, SDLoc(N), MVT::i8);
939}]>;
940
941//===----------------------------------------------------------------------===//
942// Custom Operands
943//===----------------------------------------------------------------------===//
944
945def SoppBrTarget : AsmOperandClass {
946  let Name = "SoppBrTarget";
947  let ParserMethod = "parseSOppBrTarget";
948}
949
950def sopp_brtarget : Operand<OtherVT> {
951  let EncoderMethod = "getSOPPBrEncoding";
952  let DecoderMethod = "decodeSoppBrTarget";
953  let OperandType = "OPERAND_PCREL";
954  let ParserMatchClass = SoppBrTarget;
955}
956
957def si_ga : Operand<iPTR>;
958
959def InterpSlotMatchClass : AsmOperandClass {
960  let Name = "InterpSlot";
961  let PredicateMethod = "isInterpSlot";
962  let ParserMethod = "parseInterpSlot";
963  let RenderMethod = "addImmOperands";
964}
965
966def InterpSlot : Operand<i32> {
967  let PrintMethod = "printInterpSlot";
968  let ParserMatchClass = InterpSlotMatchClass;
969  let OperandType = "OPERAND_IMMEDIATE";
970}
971
972def AttrMatchClass : AsmOperandClass {
973  let Name = "Attr";
974  let PredicateMethod = "isInterpAttr";
975  let ParserMethod = "parseInterpAttr";
976  let RenderMethod = "addImmOperands";
977}
978
979// It appears to be necessary to create a separate operand for this to
980// be able to parse attr<num> with no space.
981def Attr : Operand<i32> {
982  let PrintMethod = "printInterpAttr";
983  let ParserMatchClass = AttrMatchClass;
984  let OperandType = "OPERAND_IMMEDIATE";
985}
986
987def AttrChanMatchClass : AsmOperandClass {
988  let Name = "AttrChan";
989  let PredicateMethod = "isAttrChan";
990  let RenderMethod = "addImmOperands";
991}
992
993def AttrChan : Operand<i32> {
994  let PrintMethod = "printInterpAttrChan";
995  let ParserMatchClass = AttrChanMatchClass;
996  let OperandType = "OPERAND_IMMEDIATE";
997}
998
999def SendMsgMatchClass : AsmOperandClass {
1000  let Name = "SendMsg";
1001  let PredicateMethod = "isSendMsg";
1002  let ParserMethod = "parseSendMsgOp";
1003  let RenderMethod = "addImmOperands";
1004}
1005
1006def SwizzleMatchClass : AsmOperandClass {
1007  let Name = "Swizzle";
1008  let PredicateMethod = "isSwizzle";
1009  let ParserMethod = "parseSwizzleOp";
1010  let RenderMethod = "addImmOperands";
1011  let IsOptional = 1;
1012}
1013
1014def EndpgmMatchClass : AsmOperandClass {
1015  let Name = "EndpgmImm";
1016  let PredicateMethod = "isEndpgm";
1017  let ParserMethod = "parseEndpgmOp";
1018  let RenderMethod = "addImmOperands";
1019  let IsOptional = 1;
1020}
1021
1022def ExpTgtMatchClass : AsmOperandClass {
1023  let Name = "ExpTgt";
1024  let PredicateMethod = "isExpTgt";
1025  let ParserMethod = "parseExpTgt";
1026  let RenderMethod = "printExpTgt";
1027}
1028
1029def SWaitMatchClass : AsmOperandClass {
1030  let Name = "SWaitCnt";
1031  let RenderMethod = "addImmOperands";
1032  let ParserMethod = "parseSWaitCntOps";
1033}
1034
1035def DepCtrMatchClass : AsmOperandClass {
1036  let Name = "DepCtr";
1037  let RenderMethod = "addImmOperands";
1038  let ParserMethod = "parseDepCtrOps";
1039}
1040
1041def SDelayMatchClass : AsmOperandClass {
1042  let Name = "SDelayAlu";
1043  let RenderMethod = "addImmOperands";
1044  let ParserMethod = "parseSDelayAluOps";
1045}
1046
1047def VReg32OrOffClass : AsmOperandClass {
1048  let Name = "VReg32OrOff";
1049  let ParserMethod = "parseVReg32OrOff";
1050}
1051
1052let OperandType = "OPERAND_IMMEDIATE" in {
1053def SendMsgImm : Operand<i32> {
1054  let PrintMethod = "printSendMsg";
1055  let ParserMatchClass = SendMsgMatchClass;
1056}
1057
1058def SwizzleImm : Operand<i16> {
1059  let PrintMethod = "printSwizzle";
1060  let ParserMatchClass = SwizzleMatchClass;
1061}
1062
1063def EndpgmImm : Operand<i16> {
1064  let PrintMethod = "printEndpgm";
1065  let ParserMatchClass = EndpgmMatchClass;
1066}
1067
1068def WAIT_FLAG : Operand <i32> {
1069  let ParserMatchClass = SWaitMatchClass;
1070  let PrintMethod = "printWaitFlag";
1071}
1072
1073def DepCtrImm : Operand <i32> {
1074  let ParserMatchClass = DepCtrMatchClass;
1075  let PrintMethod = "printDepCtr";
1076}
1077
1078def DELAY_FLAG : Operand <i32> {
1079  let ParserMatchClass = SDelayMatchClass;
1080  let PrintMethod = "printDelayFlag";
1081}
1082} // End OperandType = "OPERAND_IMMEDIATE"
1083
1084include "SIInstrFormats.td"
1085include "VIInstrFormats.td"
1086
1087def BoolReg : AsmOperandClass {
1088  let Name = "BoolReg";
1089  let ParserMethod = "parseBoolReg";
1090  let RenderMethod = "addRegOperands";
1091}
1092
1093class BoolRC : RegisterOperand<SReg_1> {
1094  let ParserMatchClass = BoolReg;
1095  let DecoderMethod = "decodeBoolReg";
1096}
1097
1098def SSrc_i1 : RegisterOperand<SReg_1_XEXEC> {
1099  let ParserMatchClass = BoolReg;
1100  let DecoderMethod = "decodeBoolReg";
1101}
1102
1103def VOPDstS64orS32 : BoolRC {
1104  let PrintMethod = "printVOPDst";
1105}
1106
1107// SCSrc_i1 is the operand for pseudo instructions only.
1108// Boolean immediates shall not be exposed to codegen instructions.
1109def SCSrc_i1 : RegisterOperand<SReg_1_XEXEC> {
1110  let OperandNamespace = "AMDGPU";
1111  let OperandType = "OPERAND_REG_IMM_INT32";
1112  let ParserMatchClass = BoolReg;
1113  let DecoderMethod = "decodeBoolReg";
1114}
1115
1116// ===----------------------------------------------------------------------===//
1117// ExpSrc* Special cases for exp src operands which are printed as
1118// "off" depending on en operand.
1119// ===----------------------------------------------------------------------===//
1120
1121def ExpSrc0 : RegisterOperand<VGPR_32> {
1122  let PrintMethod = "printExpSrc0";
1123  let ParserMatchClass = VReg32OrOffClass;
1124}
1125
1126def ExpSrc1 : RegisterOperand<VGPR_32> {
1127  let PrintMethod = "printExpSrc1";
1128  let ParserMatchClass = VReg32OrOffClass;
1129}
1130
1131def ExpSrc2 : RegisterOperand<VGPR_32> {
1132  let PrintMethod = "printExpSrc2";
1133  let ParserMatchClass = VReg32OrOffClass;
1134}
1135
1136def ExpSrc3 : RegisterOperand<VGPR_32> {
1137  let PrintMethod = "printExpSrc3";
1138  let ParserMatchClass = VReg32OrOffClass;
1139}
1140
1141class SDWASrc<ValueType vt> : RegisterOperand<VS_32> {
1142  let OperandNamespace = "AMDGPU";
1143  string Type = !if(isFloatType<vt>.ret, "FP", "INT");
1144  let OperandType = "OPERAND_REG_INLINE_C_"#Type#vt.Size;
1145  let DecoderMethod = "decodeSDWASrc"#vt.Size;
1146  let EncoderMethod = "getSDWASrcEncoding";
1147}
1148
1149def SDWASrc_i32 : SDWASrc<i32>;
1150def SDWASrc_i16 : SDWASrc<i16>;
1151def SDWASrc_f32 : SDWASrc<f32>;
1152def SDWASrc_f16 : SDWASrc<f16>;
1153
1154def SDWAVopcDst : BoolRC {
1155  let OperandNamespace = "AMDGPU";
1156  let OperandType = "OPERAND_SDWA_VOPC_DST";
1157  let EncoderMethod = "getSDWAVopcDstEncoding";
1158  let DecoderMethod = "decodeSDWAVopcDst";
1159  let PrintMethod = "printVOPDst";
1160}
1161
1162class NamedMatchClass<string CName, bit Optional = 1> : AsmOperandClass {
1163  let Name = "Imm"#CName;
1164  let PredicateMethod = "is"#CName;
1165  let ParserMethod = !if(Optional, "parseOptionalOperand", "parse"#CName);
1166  let RenderMethod = "addImmOperands";
1167  let IsOptional = Optional;
1168  let DefaultMethod = !if(Optional, "default"#CName, ?);
1169}
1170
1171class NamedOperandBit<string Name, AsmOperandClass MatchClass> : Operand<i1> {
1172  let PrintMethod = "print"#Name;
1173  let ParserMatchClass = MatchClass;
1174}
1175
1176class NamedOperandBit_0<string Name, AsmOperandClass MatchClass> :
1177  OperandWithDefaultOps<i1, (ops (i1 0))> {
1178  let PrintMethod = "print"#Name;
1179  let ParserMatchClass = MatchClass;
1180}
1181
1182class NamedOperandBit_1<string Name, AsmOperandClass MatchClass> :
1183  OperandWithDefaultOps<i1, (ops (i1 1))> {
1184  let PrintMethod = "print"#Name;
1185  let ParserMatchClass = MatchClass;
1186}
1187
1188class NamedOperandU8<string Name, AsmOperandClass MatchClass> : Operand<i8> {
1189  let PrintMethod = "print"#Name;
1190  let ParserMatchClass = MatchClass;
1191}
1192
1193class NamedOperandU16<string Name, AsmOperandClass MatchClass> : Operand<i16> {
1194  let PrintMethod = "print"#Name;
1195  let ParserMatchClass = MatchClass;
1196}
1197
1198class NamedOperandU32<string Name, AsmOperandClass MatchClass> : Operand<i32> {
1199  let PrintMethod = "print"#Name;
1200  let ParserMatchClass = MatchClass;
1201}
1202
1203class NamedOperandU32_0<string Name, AsmOperandClass MatchClass> :
1204  OperandWithDefaultOps<i32, (ops (i32 0))> {
1205  let PrintMethod = "print"#Name;
1206  let ParserMatchClass = MatchClass;
1207}
1208
1209class NamedOperandU32Default0<string Name, AsmOperandClass MatchClass> :
1210  OperandWithDefaultOps<i32, (ops (i32 0))> {
1211  let PrintMethod = "print"#Name;
1212  let ParserMatchClass = MatchClass;
1213}
1214
1215class NamedOperandU32Default1<string Name, AsmOperandClass MatchClass> :
1216  OperandWithDefaultOps<i32, (ops (i32 1))> {
1217  let PrintMethod = "print"#Name;
1218  let ParserMatchClass = MatchClass;
1219}
1220
1221let OperandType = "OPERAND_IMMEDIATE" in {
1222
1223def offen : NamedOperandBit<"Offen", NamedMatchClass<"Offen">>;
1224def idxen : NamedOperandBit<"Idxen", NamedMatchClass<"Idxen">>;
1225def addr64 : NamedOperandBit<"Addr64", NamedMatchClass<"Addr64">>;
1226
1227def flat_offset : NamedOperandU16<"FlatOffset", NamedMatchClass<"FlatOffset">>;
1228def offset : NamedOperandU16<"Offset", NamedMatchClass<"Offset">>;
1229def offset0 : NamedOperandU8<"Offset0", NamedMatchClass<"Offset0">>;
1230def offset1 : NamedOperandU8<"Offset1", NamedMatchClass<"Offset1">>;
1231
1232def gds : NamedOperandBit<"GDS", NamedMatchClass<"GDS">>;
1233
1234def omod : NamedOperandU32<"OModSI", NamedMatchClass<"OModSI">>;
1235def omod0 : NamedOperandU32_0<"OModSI", NamedMatchClass<"OModSI">>;
1236
1237// We need to make the cases with a default of 0 distinct from no
1238// default to help deal with some cases where the operand appears
1239// before a mandatory operand.
1240def clampmod : NamedOperandBit<"ClampSI", NamedMatchClass<"ClampSI">>;
1241def clampmod0 : NamedOperandBit_0<"ClampSI", NamedMatchClass<"ClampSI">>;
1242def highmod : NamedOperandBit<"High", NamedMatchClass<"High">>;
1243
1244def CPol : NamedOperandU32<"CPol", NamedMatchClass<"CPol">>;
1245def CPol_0 : NamedOperandU32Default0<"CPol", NamedMatchClass<"CPol">>;
1246def CPol_GLC1 : NamedOperandU32Default1<"CPol", NamedMatchClass<"CPol">>;
1247
1248def TFE : NamedOperandBit<"TFE", NamedMatchClass<"TFE">>;
1249def TFE_0 : NamedOperandBit_0<"TFE", NamedMatchClass<"TFE">>;
1250def SWZ : NamedOperandBit<"SWZ", NamedMatchClass<"SWZ">>;
1251def SWZ_0 : NamedOperandBit_0<"SWZ", NamedMatchClass<"SWZ">>;
1252def UNorm : NamedOperandBit<"UNorm", NamedMatchClass<"UNorm">>;
1253def DA : NamedOperandBit<"DA", NamedMatchClass<"DA">>;
1254def R128A16 : NamedOperandBit<"R128A16", NamedMatchClass<"R128A16">>;
1255def GFX10A16 : NamedOperandBit<"GFX10A16", NamedMatchClass<"GFX10A16">>;
1256def D16 : NamedOperandBit<"D16", NamedMatchClass<"D16">>;
1257def LWE : NamedOperandBit<"LWE", NamedMatchClass<"LWE">>;
1258def exp_compr : NamedOperandBit<"ExpCompr", NamedMatchClass<"ExpCompr">>;
1259def exp_vm : NamedOperandBit<"ExpVM", NamedMatchClass<"ExpVM">>;
1260
1261def FORMAT : NamedOperandU8<"FORMAT", NamedMatchClass<"FORMAT", 0>>;
1262
1263def DMask : NamedOperandU16<"DMask", NamedMatchClass<"DMask">>;
1264def Dim : NamedOperandU8<"Dim", NamedMatchClass<"Dim", 0>>;
1265
1266def dpp8 : NamedOperandU32<"DPP8", NamedMatchClass<"DPP8", 0>>;
1267
1268def dpp_ctrl : NamedOperandU32<"DPPCtrl", NamedMatchClass<"DPPCtrl", 0>>;
1269def row_mask : NamedOperandU32<"RowMask", NamedMatchClass<"RowMask">>;
1270def bank_mask : NamedOperandU32<"BankMask", NamedMatchClass<"BankMask">>;
1271def bound_ctrl : NamedOperandBit<"BoundCtrl", NamedMatchClass<"BoundCtrl">>;
1272def FI : NamedOperandU32<"FI", NamedMatchClass<"FI">>;
1273
1274def dst_sel : NamedOperandU32<"SDWADstSel", NamedMatchClass<"SDWADstSel">>;
1275def src0_sel : NamedOperandU32<"SDWASrc0Sel", NamedMatchClass<"SDWASrc0Sel">>;
1276def src1_sel : NamedOperandU32<"SDWASrc1Sel", NamedMatchClass<"SDWASrc1Sel">>;
1277def dst_unused : NamedOperandU32<"SDWADstUnused", NamedMatchClass<"SDWADstUnused">>;
1278
1279def op_sel0 : NamedOperandU32Default0<"OpSel", NamedMatchClass<"OpSel">>;
1280def op_sel_hi0 : NamedOperandU32Default0<"OpSelHi", NamedMatchClass<"OpSelHi">>;
1281def neg_lo0 : NamedOperandU32Default0<"NegLo", NamedMatchClass<"NegLo">>;
1282def neg_hi0 : NamedOperandU32Default0<"NegHi", NamedMatchClass<"NegHi">>;
1283
1284def blgp : NamedOperandU32<"BLGP", NamedMatchClass<"BLGP">>;
1285def cbsz : NamedOperandU32<"CBSZ", NamedMatchClass<"CBSZ">>;
1286def abid : NamedOperandU32<"ABID", NamedMatchClass<"ABID">>;
1287
1288def hwreg : NamedOperandU32<"Hwreg", NamedMatchClass<"Hwreg", 0>>;
1289
1290def exp_tgt : NamedOperandU32<"ExpTgt", NamedMatchClass<"ExpTgt", 0>> {
1291
1292}
1293
1294def wait_vdst : NamedOperandU8<"WaitVDST", NamedMatchClass<"WaitVDST">>;
1295def wait_exp : NamedOperandU8<"WaitEXP", NamedMatchClass<"WaitEXP">>;
1296
1297} // End OperandType = "OPERAND_IMMEDIATE"
1298
1299class KImmMatchClass<int size> : AsmOperandClass {
1300  let Name = "KImmFP"#size;
1301  let PredicateMethod = "isKImmFP"#size;
1302  let ParserMethod = "parseImm";
1303  let RenderMethod = "addKImmFP"#size#"Operands";
1304}
1305
1306class kimmOperand<ValueType vt> : Operand<vt> {
1307  let OperandNamespace = "AMDGPU";
1308  let OperandType = "OPERAND_KIMM"#vt.Size;
1309  let PrintMethod = "printU"#vt.Size#"ImmOperand";
1310  let ParserMatchClass = !cast<AsmOperandClass>("KImmFP"#vt.Size#"MatchClass");
1311  let DecoderMethod = "decodeOperand_f"#vt.Size#"kimm";
1312}
1313
1314// 32-bit VALU immediate operand that uses the constant bus.
1315def KImmFP32MatchClass : KImmMatchClass<32>;
1316def f32kimm : kimmOperand<i32>;
1317
1318// 32-bit VALU immediate operand with a 16-bit value that uses the
1319// constant bus.
1320def KImmFP16MatchClass : KImmMatchClass<16>;
1321def f16kimm : kimmOperand<i16>;
1322
1323class FPInputModsMatchClass <int opSize> : AsmOperandClass {
1324  let Name = "RegOrImmWithFP"#opSize#"InputMods";
1325  let ParserMethod = "parseRegOrImmWithFPInputMods";
1326  let PredicateMethod = "isRegOrImmWithFP"#opSize#"InputMods";
1327}
1328
1329def FP16InputModsMatchClass : FPInputModsMatchClass<16>;
1330def FP32InputModsMatchClass : FPInputModsMatchClass<32>;
1331def FP64InputModsMatchClass : FPInputModsMatchClass<64>;
1332
1333class InputMods <AsmOperandClass matchClass> : Operand <i32> {
1334  let OperandNamespace = "AMDGPU";
1335  let OperandType = "OPERAND_INPUT_MODS";
1336  let ParserMatchClass = matchClass;
1337}
1338
1339class FPInputMods <FPInputModsMatchClass matchClass> : InputMods <matchClass> {
1340  let PrintMethod = "printOperandAndFPInputMods";
1341}
1342
1343def FP16InputMods : FPInputMods<FP16InputModsMatchClass>;
1344def FP32InputMods : FPInputMods<FP32InputModsMatchClass>;
1345def FP64InputMods : FPInputMods<FP64InputModsMatchClass>;
1346
1347class IntInputModsMatchClass <int opSize> : AsmOperandClass {
1348  let Name = "RegOrImmWithInt"#opSize#"InputMods";
1349  let ParserMethod = "parseRegOrImmWithIntInputMods";
1350  let PredicateMethod = "isRegOrImmWithInt"#opSize#"InputMods";
1351}
1352def Int32InputModsMatchClass : IntInputModsMatchClass<32>;
1353def Int64InputModsMatchClass : IntInputModsMatchClass<64>;
1354
1355class IntInputMods <IntInputModsMatchClass matchClass> : InputMods <matchClass> {
1356  let PrintMethod = "printOperandAndIntInputMods";
1357}
1358def Int32InputMods : IntInputMods<Int32InputModsMatchClass>;
1359def Int64InputMods : IntInputMods<Int64InputModsMatchClass>;
1360
1361class OpSelModsMatchClass : AsmOperandClass {
1362  let Name = "OpSelMods";
1363  let ParserMethod = "parseRegOrImm";
1364  let PredicateMethod = "isRegOrImm";
1365}
1366
1367def IntOpSelModsMatchClass : OpSelModsMatchClass;
1368def IntOpSelMods : InputMods<IntOpSelModsMatchClass>;
1369
1370class FPSDWAInputModsMatchClass <int opSize> : AsmOperandClass {
1371  let Name = "SDWAWithFP"#opSize#"InputMods";
1372  let ParserMethod = "parseRegOrImmWithFPInputMods";
1373  let PredicateMethod = "isSDWAFP"#opSize#"Operand";
1374}
1375
1376def FP16SDWAInputModsMatchClass : FPSDWAInputModsMatchClass<16>;
1377def FP32SDWAInputModsMatchClass : FPSDWAInputModsMatchClass<32>;
1378
1379class FPSDWAInputMods <FPSDWAInputModsMatchClass matchClass> :
1380  InputMods <matchClass> {
1381  let PrintMethod = "printOperandAndFPInputMods";
1382}
1383
1384def FP16SDWAInputMods : FPSDWAInputMods<FP16SDWAInputModsMatchClass>;
1385def FP32SDWAInputMods : FPSDWAInputMods<FP32SDWAInputModsMatchClass>;
1386
1387def FPVRegInputModsMatchClass : AsmOperandClass {
1388  let Name = "VRegWithFPInputMods";
1389  let ParserMethod = "parseRegWithFPInputMods";
1390  let PredicateMethod = "isVRegWithInputMods";
1391}
1392
1393def FPVRegInputMods : InputMods <FPVRegInputModsMatchClass> {
1394  let PrintMethod = "printOperandAndFPInputMods";
1395}
1396
1397class IntSDWAInputModsMatchClass <int opSize> : AsmOperandClass {
1398  let Name = "SDWAWithInt"#opSize#"InputMods";
1399  let ParserMethod = "parseRegOrImmWithIntInputMods";
1400  let PredicateMethod = "isSDWAInt"#opSize#"Operand";
1401}
1402
1403def Int16SDWAInputModsMatchClass : IntSDWAInputModsMatchClass<16>;
1404def Int32SDWAInputModsMatchClass : IntSDWAInputModsMatchClass<32>;
1405
1406class IntSDWAInputMods <IntSDWAInputModsMatchClass matchClass> :
1407  InputMods <matchClass> {
1408  let PrintMethod = "printOperandAndIntInputMods";
1409}
1410
1411def Int16SDWAInputMods : IntSDWAInputMods<Int16SDWAInputModsMatchClass>;
1412def Int32SDWAInputMods : IntSDWAInputMods<Int32SDWAInputModsMatchClass>;
1413
1414def IntVRegInputModsMatchClass : AsmOperandClass {
1415  let Name = "VRegWithIntInputMods";
1416  let ParserMethod = "parseRegWithIntInputMods";
1417  let PredicateMethod = "isVRegWithInputMods";
1418}
1419
1420def IntVRegInputMods : InputMods <IntVRegInputModsMatchClass> {
1421  let PrintMethod = "printOperandAndIntInputMods";
1422}
1423
1424class PackedFPInputModsMatchClass <int opSize> : AsmOperandClass {
1425  let Name = "PackedFP"#opSize#"InputMods";
1426  let ParserMethod = "parseRegOrImm";
1427  let PredicateMethod = "isRegOrImm";
1428//  let PredicateMethod = "isPackedFP"#opSize#"InputMods";
1429}
1430
1431class PackedIntInputModsMatchClass <int opSize> : AsmOperandClass {
1432  let Name = "PackedInt"#opSize#"InputMods";
1433  let ParserMethod = "parseRegOrImm";
1434  let PredicateMethod = "isRegOrImm";
1435//  let PredicateMethod = "isPackedInt"#opSize#"InputMods";
1436}
1437
1438def PackedF16InputModsMatchClass : PackedFPInputModsMatchClass<16>;
1439def PackedI16InputModsMatchClass : PackedIntInputModsMatchClass<16>;
1440
1441class PackedFPInputMods <PackedFPInputModsMatchClass matchClass> : InputMods <matchClass> {
1442//  let PrintMethod = "printPackedFPInputMods";
1443}
1444
1445class PackedIntInputMods <PackedIntInputModsMatchClass matchClass> : InputMods <matchClass> {
1446  //let PrintMethod = "printPackedIntInputMods";
1447}
1448
1449def PackedF16InputMods : PackedFPInputMods<PackedF16InputModsMatchClass>;
1450def PackedI16InputMods : PackedIntInputMods<PackedI16InputModsMatchClass>;
1451
1452//===----------------------------------------------------------------------===//
1453// Complex patterns
1454//===----------------------------------------------------------------------===//
1455
1456def DS1Addr1Offset : ComplexPattern<iPTR, 2, "SelectDS1Addr1Offset">;
1457def DS64Bit4ByteAligned : ComplexPattern<iPTR, 3, "SelectDS64Bit4ByteAligned">;
1458def DS128Bit8ByteAligned : ComplexPattern<iPTR, 3, "SelectDS128Bit8ByteAligned">;
1459
1460def MOVRELOffset : ComplexPattern<iPTR, 2, "SelectMOVRELOffset">;
1461
1462def VOP3Mods0 : ComplexPattern<untyped, 4, "SelectVOP3Mods0">;
1463def VOP3Mods  : ComplexPattern<untyped, 2, "SelectVOP3Mods">;
1464def VOP3NoMods : ComplexPattern<untyped, 1, "SelectVOP3NoMods">;
1465// VOP3Mods, but the input source is known to never be NaN.
1466def VOP3Mods_nnan : ComplexPattern<fAny, 2, "SelectVOP3Mods_NNaN">;
1467
1468def VOP3OMods : ComplexPattern<untyped, 3, "SelectVOP3OMods">;
1469
1470def VOP3PMods  : ComplexPattern<untyped, 2, "SelectVOP3PMods">;
1471
1472def VOP3PModsDOT  : ComplexPattern<untyped, 2, "SelectVOP3PModsDOT">;
1473
1474def VOP3OpSel  : ComplexPattern<untyped, 2, "SelectVOP3OpSel">;
1475
1476def VOP3OpSelMods  : ComplexPattern<untyped, 2, "SelectVOP3OpSelMods">;
1477
1478def VOP3PMadMixMods  : ComplexPattern<untyped, 2, "SelectVOP3PMadMixMods">;
1479
1480def VINTERPMods  : ComplexPattern<untyped, 2, "SelectVINTERPMods">;
1481def VINTERPModsHi  : ComplexPattern<untyped, 2, "SelectVINTERPModsHi">;
1482
1483//===----------------------------------------------------------------------===//
1484// SI assembler operands
1485//===----------------------------------------------------------------------===//
1486
1487def SIOperand {
1488  int ZERO = 0x80;
1489  int VCC = 0x6A;
1490  int FLAT_SCR = 0x68;
1491}
1492
1493// This should be kept in sync with SISrcMods enum
1494def SRCMODS {
1495  int NONE = 0;
1496  int NEG = 1;
1497  int ABS = 2;
1498  int NEG_ABS = 3;
1499
1500  int NEG_HI = ABS;
1501  int OP_SEL_0 = 4;
1502  int OP_SEL_1 = 8;
1503  int DST_OP_SEL = 8;
1504}
1505
1506def DSTCLAMP {
1507  int NONE = 0;
1508  int ENABLE = 1;
1509}
1510
1511def DSTOMOD {
1512  int NONE = 0;
1513}
1514
1515def HWREG {
1516  int MODE = 1;
1517  int STATUS = 2;
1518  int TRAPSTS = 3;
1519  int HW_ID = 4;
1520  int GPR_ALLOC = 5;
1521  int LDS_ALLOC = 6;
1522  int IB_STS = 7;
1523  int MEM_BASES = 15;
1524  int TBA_LO = 16;
1525  int TBA_HI = 17;
1526  int TMA_LO = 18;
1527  int TMA_HI = 19;
1528  int FLAT_SCR_LO = 20;
1529  int FLAT_SCR_HI = 21;
1530  int XNACK_MASK = 22;
1531  int POPS_PACKER = 25;
1532  int SHADER_CYCLES = 29;
1533}
1534
1535class getHwRegImm<int Reg, int Offset = 0, int Size = 32> {
1536  int ret = !and(!or(Reg,
1537                     !shl(Offset, 6),
1538                     !shl(!add(Size, -1), 11)), 65535);
1539}
1540
1541//===----------------------------------------------------------------------===//
1542//
1543// SI Instruction multiclass helpers.
1544//
1545// Instructions with _32 take 32-bit operands.
1546// Instructions with _64 take 64-bit operands.
1547//
1548// VOP_* instructions can use either a 32-bit or 64-bit encoding.  The 32-bit
1549// encoding is the standard encoding, but instruction that make use of
1550// any of the instruction modifiers must use the 64-bit encoding.
1551//
1552// Instructions with _e32 use the 32-bit encoding.
1553// Instructions with _e64 use the 64-bit encoding.
1554//
1555//===----------------------------------------------------------------------===//
1556
1557class SIMCInstr <string pseudo, int subtarget> {
1558  string PseudoInstr = pseudo;
1559  int Subtarget = subtarget;
1560}
1561
1562//===----------------------------------------------------------------------===//
1563// Vector ALU classes
1564//===----------------------------------------------------------------------===//
1565
1566class getNumSrcArgs<ValueType Src0, ValueType Src1, ValueType Src2> {
1567  int ret =
1568    !if (!eq(Src0.Value, untyped.Value),      0,
1569      !if (!eq(Src1.Value, untyped.Value),    1,   // VOP1
1570         !if (!eq(Src2.Value, untyped.Value), 2,   // VOP2
1571                                              3))); // VOP3
1572}
1573
1574// Returns the register class to use for the destination of VOP[123C]
1575// instructions for the given VT.
1576class getVALUDstForVT<ValueType VT> {
1577  RegisterOperand ret = !if(!eq(VT.Size, 32), VOPDstOperand<VGPR_32>,
1578                          !if(!eq(VT.Size, 128), VOPDstOperand<VReg_128>,
1579                            !if(!eq(VT.Size, 64), VOPDstOperand<VReg_64>,
1580                              !if(!eq(VT.Size, 16), VOPDstOperand<VGPR_32>,
1581                              VOPDstS64orS32)))); // else VT == i1
1582}
1583
1584// Returns the register class to use for the destination of VOP[12C]
1585// instructions with SDWA extension
1586class getSDWADstForVT<ValueType VT> {
1587  RegisterOperand ret = !if(!eq(VT.Size, 1),
1588                            SDWAVopcDst, // VOPC
1589                            VOPDstOperand<VGPR_32>); // VOP1/2 32-bit dst
1590}
1591
1592// Returns the register class to use for source 0 of VOP[12C]
1593// instructions for the given VT.
1594class getVOPSrc0ForVT<ValueType VT> {
1595  bit isFP = isFloatType<VT>.ret;
1596
1597  RegisterOperand ret =
1598    !if(isFP,
1599      !if(!eq(VT.Size, 64),
1600         VSrc_f64,
1601         !if(!eq(VT.Value, f16.Value),
1602            VSrc_f16,
1603            !if(!eq(VT.Value, v2f16.Value),
1604               VSrc_v2f16,
1605               !if(!eq(VT.Value, v4f16.Value),
1606                 AVSrc_64,
1607                 VSrc_f32
1608               )
1609            )
1610         )
1611       ),
1612       !if(!eq(VT.Size, 64),
1613          VSrc_b64,
1614          !if(!eq(VT.Value, i16.Value),
1615             VSrc_b16,
1616             !if(!eq(VT.Value, v2i16.Value),
1617                VSrc_v2b16,
1618                VSrc_b32
1619             )
1620          )
1621       )
1622    );
1623}
1624
1625class getSOPSrcForVT<ValueType VT> {
1626  RegisterOperand ret = !if(!eq(VT.Size, 64), SSrc_b64, SSrc_b32);
1627}
1628
1629// Returns the vreg register class to use for source operand given VT
1630class getVregSrcForVT<ValueType VT> {
1631  RegisterClass ret = !if(!eq(VT.Size, 128), VReg_128,
1632                        !if(!eq(VT.Size, 96), VReg_96,
1633                          !if(!eq(VT.Size, 64), VReg_64,
1634                            !if(!eq(VT.Size, 48), VReg_64,
1635                              VGPR_32))));
1636}
1637
1638class getSDWASrcForVT <ValueType VT> {
1639  bit isFP = isFloatType<VT>.ret;
1640  RegisterOperand retFlt = !if(!eq(VT.Size, 16), SDWASrc_f16, SDWASrc_f32);
1641  RegisterOperand retInt = !if(!eq(VT.Size, 16), SDWASrc_i16, SDWASrc_i32);
1642  RegisterOperand ret = !if(isFP, retFlt, retInt);
1643}
1644
1645// Returns the register class to use for sources of VOP3 instructions for the
1646// given VT.
1647class getVOP3SrcForVT<ValueType VT> {
1648  bit isFP = isFloatType<VT>.ret;
1649  RegisterOperand ret =
1650  !if(!eq(VT.Size, 128),
1651     VSrc_128,
1652     !if(!eq(VT.Size, 64),
1653        !if(isFP,
1654           !if(!eq(VT.Value, v2f32.Value),
1655               VSrc_v2f32,
1656               VSrc_f64),
1657           !if(!eq(VT.Value, v2i32.Value),
1658               VSrc_v2b32,
1659           VSrc_b64)),
1660        !if(!eq(VT.Value, i1.Value),
1661           SSrc_i1,
1662           !if(isFP,
1663              !if(!eq(VT.Value, f16.Value),
1664                 VSrc_f16,
1665                 !if(!eq(VT.Value, v2f16.Value),
1666                    VSrc_v2f16,
1667                    !if(!eq(VT.Value, v4f16.Value),
1668                      AVSrc_64,
1669                      VSrc_f32
1670                    )
1671                 )
1672              ),
1673              !if(!eq(VT.Value, i16.Value),
1674                 VSrc_b16,
1675                 !if(!eq(VT.Value, v2i16.Value),
1676                    VSrc_v2b16,
1677                    VSrc_b32
1678                 )
1679              )
1680           )
1681        )
1682     )
1683  );
1684}
1685
1686// Float or packed int
1687class isModifierType<ValueType SrcVT> {
1688  bit ret = !or(!eq(SrcVT.Value, f16.Value),
1689                !eq(SrcVT.Value, f32.Value),
1690                !eq(SrcVT.Value, f64.Value),
1691                !eq(SrcVT.Value, v2f16.Value),
1692                !eq(SrcVT.Value, v2i16.Value),
1693                !eq(SrcVT.Value, v2f32.Value),
1694                !eq(SrcVT.Value, v2i32.Value));
1695}
1696
1697// Return type of input modifiers operand for specified input operand
1698class getSrcMod <ValueType VT, bit EnableF32SrcMods> {
1699  bit isFP = isFloatType<VT>.ret;
1700  bit isPacked = isPackedType<VT>.ret;
1701  Operand ret =  !if(!eq(VT.Size, 64),
1702                     !if(isFP, FP64InputMods, Int64InputMods),
1703                       !if(isFP,
1704                         !if(!eq(VT.Value, f16.Value),
1705                            FP16InputMods,
1706                            FP32InputMods
1707                          ),
1708                         !if(EnableF32SrcMods, FP32InputMods, Int32InputMods))
1709                     );
1710}
1711
1712class getOpSelMod <ValueType VT> {
1713  Operand ret = !if(!eq(VT.Value, f16.Value), FP16InputMods, IntOpSelMods);
1714}
1715
1716// Return type of input modifiers operand specified input operand for DPP
1717class getSrcModDPP <ValueType VT> {
1718  bit isFP = isFloatType<VT>.ret;
1719  Operand ret = !if(isFP, FPVRegInputMods, IntVRegInputMods);
1720}
1721
1722// Return type of input modifiers operand specified input operand for SDWA
1723class getSrcModSDWA <ValueType VT> {
1724  Operand ret = !if(!eq(VT.Value, f16.Value), FP16SDWAInputMods,
1725                !if(!eq(VT.Value, f32.Value), FP32SDWAInputMods,
1726                !if(!eq(VT.Value, i16.Value), Int16SDWAInputMods,
1727                Int32SDWAInputMods)));
1728}
1729
1730// Returns the input arguments for VOP[12C] instructions for the given SrcVT.
1731class getIns32 <RegisterOperand Src0RC, RegisterClass Src1RC, int NumSrcArgs> {
1732  dag ret = !if(!eq(NumSrcArgs, 1), (ins Src0RC:$src0),               // VOP1
1733            !if(!eq(NumSrcArgs, 2), (ins Src0RC:$src0, Src1RC:$src1), // VOP2
1734                                    (ins)));
1735}
1736
1737// Returns the input arguments for VOP3 instructions for the given SrcVT.
1738class getIns64 <RegisterOperand Src0RC, RegisterOperand Src1RC,
1739                RegisterOperand Src2RC, int NumSrcArgs,
1740                bit HasClamp, bit HasModifiers, bit HasSrc2Mods, bit HasOMod,
1741                Operand Src0Mod, Operand Src1Mod, Operand Src2Mod> {
1742
1743  dag ret =
1744    !if (!eq(NumSrcArgs, 0),
1745      // VOP1 without input operands (V_NOP, V_CLREXCP)
1746      (ins),
1747      /* else */
1748    !if (!eq(NumSrcArgs, 1),
1749      !if (HasModifiers,
1750        // VOP1 with modifiers
1751        !if(HasOMod,
1752          (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
1753               clampmod0:$clamp, omod0:$omod),
1754          (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
1755               clampmod0:$clamp))
1756      /* else */,
1757        // VOP1 without modifiers
1758        !if (HasClamp,
1759          (ins Src0RC:$src0, clampmod0:$clamp),
1760          (ins Src0RC:$src0))
1761      /* endif */ ),
1762    !if (!eq(NumSrcArgs, 2),
1763      !if (HasModifiers,
1764        // VOP 2 with modifiers
1765        !if(HasOMod,
1766          (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
1767               Src1Mod:$src1_modifiers, Src1RC:$src1,
1768               clampmod0:$clamp, omod0:$omod),
1769           (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
1770               Src1Mod:$src1_modifiers, Src1RC:$src1,
1771               clampmod0:$clamp))
1772      /* else */,
1773        // VOP2 without modifiers
1774        !if (HasClamp,
1775          (ins Src0RC:$src0, Src1RC:$src1, clampmod0:$clamp),
1776          (ins Src0RC:$src0, Src1RC:$src1))
1777
1778      /* endif */ )
1779    /* NumSrcArgs == 3 */,
1780      !if (HasModifiers,
1781        !if (HasSrc2Mods,
1782          // VOP3 with modifiers
1783          !if (HasOMod,
1784            (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
1785                 Src1Mod:$src1_modifiers, Src1RC:$src1,
1786                 Src2Mod:$src2_modifiers, Src2RC:$src2,
1787                 clampmod0:$clamp, omod0:$omod),
1788            !if (HasClamp,
1789              (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
1790                   Src1Mod:$src1_modifiers, Src1RC:$src1,
1791                   Src2Mod:$src2_modifiers, Src2RC:$src2,
1792                   clampmod0:$clamp),
1793              (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
1794                   Src1Mod:$src1_modifiers, Src1RC:$src1,
1795                   Src2Mod:$src2_modifiers, Src2RC:$src2))),
1796          // VOP3 with modifiers except src2
1797          !if (HasOMod,
1798            (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
1799                 Src1Mod:$src1_modifiers, Src1RC:$src1,
1800                 Src2RC:$src2, clampmod0:$clamp, omod0:$omod),
1801            !if (HasClamp,
1802              (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
1803                   Src1Mod:$src1_modifiers, Src1RC:$src1,
1804                   Src2RC:$src2, clampmod0:$clamp),
1805              (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
1806                   Src1Mod:$src1_modifiers, Src1RC:$src1,
1807                   Src2RC:$src2))))
1808      /* else */,
1809        // VOP3 without modifiers
1810        !if (HasClamp,
1811          (ins Src0RC:$src0, Src1RC:$src1, Src2RC:$src2, clampmod0:$clamp),
1812          (ins Src0RC:$src0, Src1RC:$src1, Src2RC:$src2))
1813      /* endif */ ))));
1814}
1815
1816class getInsVOP3Base<RegisterOperand Src0RC, RegisterOperand Src1RC,
1817                RegisterOperand Src2RC, int NumSrcArgs,
1818                bit HasClamp, bit HasModifiers, bit HasSrc2Mods, bit HasOMod,
1819                Operand Src0Mod, Operand Src1Mod, Operand Src2Mod, bit HasOpSel,
1820                bit IsVOP3P> {
1821  // getInst64 handles clamp and omod. implicit mutex between vop3p and omod
1822  dag base = getIns64 <Src0RC, Src1RC, Src2RC, NumSrcArgs,
1823                HasClamp, HasModifiers, HasSrc2Mods, HasOMod,
1824                Src0Mod, Src1Mod, Src2Mod>.ret;
1825  dag opsel = (ins op_sel0:$op_sel);
1826  dag vop3pFields = (ins op_sel_hi0:$op_sel_hi, neg_lo0:$neg_lo, neg_hi0:$neg_hi);
1827  dag ret = !con(base,
1828                 !if(HasOpSel, opsel,(ins)),
1829                 !if(IsVOP3P, vop3pFields,(ins)));
1830}
1831
1832class getInsVOP3P <RegisterOperand Src0RC, RegisterOperand Src1RC,
1833                   RegisterOperand Src2RC, int NumSrcArgs, bit HasClamp,
1834                   Operand Src0Mod, Operand Src1Mod, Operand Src2Mod> {
1835  dag ret = getInsVOP3Base<Src0RC, Src1RC, Src2RC, NumSrcArgs,
1836                    HasClamp, 1/*HasModifiers*/, 1/*HasSrc2Mods*/,
1837                    0/*HasOMod*/, Src0Mod, Src1Mod, Src2Mod,
1838                    1/*HasOpSel*/, 1/*IsVOP3P*/>.ret;
1839}
1840
1841class getInsVOP3OpSel <RegisterOperand Src0RC, RegisterOperand Src1RC,
1842                       RegisterOperand Src2RC, int NumSrcArgs,
1843                       bit HasClamp, bit HasOMod,
1844                       Operand Src0Mod, Operand Src1Mod, Operand Src2Mod> {
1845  dag ret = getInsVOP3Base<Src0RC, Src1RC,
1846                    Src2RC, NumSrcArgs,
1847                    HasClamp, 1/*HasModifiers*/, 1/*HasSrc2Mods*/, HasOMod,
1848                    Src0Mod, Src1Mod, Src2Mod, 1/*HasOpSel*/, 0>.ret;
1849}
1850
1851class getInsDPPBase <RegisterOperand OldRC, RegisterClass Src0RC, RegisterClass Src1RC,
1852                 int NumSrcArgs, bit HasModifiers,
1853                 Operand Src0Mod, Operand Src1Mod> {
1854
1855  dag ret = !if (!eq(NumSrcArgs, 0),
1856                // VOP1 without input operands (V_NOP)
1857                (ins ),
1858            !if (!eq(NumSrcArgs, 1),
1859              !if (HasModifiers,
1860                // VOP1_DPP with modifiers
1861                (ins OldRC:$old, Src0Mod:$src0_modifiers,
1862                     Src0RC:$src0)
1863              /* else */,
1864                // VOP1_DPP without modifiers
1865                (ins OldRC:$old, Src0RC:$src0)
1866              /* endif */),
1867              !if (HasModifiers,
1868                // VOP2_DPP with modifiers
1869                (ins OldRC:$old,
1870                     Src0Mod:$src0_modifiers, Src0RC:$src0,
1871                     Src1Mod:$src1_modifiers, Src1RC:$src1)
1872              /* else */,
1873                // VOP2_DPP without modifiers
1874                (ins OldRC:$old,
1875                     Src0RC:$src0, Src1RC:$src1)
1876                )));
1877}
1878
1879class getInsDPP <RegisterOperand OldRC, RegisterClass Src0RC, RegisterClass Src1RC,
1880                 int NumSrcArgs, bit HasModifiers,
1881                 Operand Src0Mod, Operand Src1Mod> {
1882  dag ret = !con(getInsDPPBase<OldRC, Src0RC, Src1RC, NumSrcArgs,
1883                               HasModifiers, Src0Mod, Src1Mod>.ret,
1884                 (ins dpp_ctrl:$dpp_ctrl, row_mask:$row_mask,
1885                  bank_mask:$bank_mask, bound_ctrl:$bound_ctrl));
1886}
1887
1888class getInsDPP16 <RegisterOperand OldRC, RegisterClass Src0RC, RegisterClass Src1RC,
1889                 int NumSrcArgs, bit HasModifiers,
1890                 Operand Src0Mod, Operand Src1Mod> {
1891  dag ret = !con(getInsDPP<OldRC, Src0RC, Src1RC, NumSrcArgs,
1892                           HasModifiers, Src0Mod, Src1Mod>.ret,
1893                 (ins FI:$fi));
1894}
1895
1896class getInsDPP8 <RegisterOperand OldRC, RegisterClass Src0RC, RegisterClass Src1RC,
1897                 int NumSrcArgs, bit HasModifiers,
1898                 Operand Src0Mod, Operand Src1Mod> {
1899  dag ret = !con(getInsDPPBase<OldRC, Src0RC, Src1RC, NumSrcArgs,
1900                               HasModifiers, Src0Mod, Src1Mod>.ret,
1901                 (ins dpp8:$dpp8, FI:$fi));
1902}
1903
1904
1905// Ins for SDWA
1906class getInsSDWA <RegisterOperand Src0RC, RegisterOperand Src1RC, int NumSrcArgs,
1907                  bit HasSDWAOMod, Operand Src0Mod, Operand Src1Mod,
1908                  ValueType DstVT> {
1909
1910  dag ret = !if(!eq(NumSrcArgs, 0),
1911               // VOP1 without input operands (V_NOP)
1912               (ins),
1913            !if(!eq(NumSrcArgs, 1),
1914               // VOP1
1915               !if(!not(HasSDWAOMod),
1916                  // VOP1_SDWA without omod
1917                  (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
1918                       clampmod:$clamp,
1919                       dst_sel:$dst_sel, dst_unused:$dst_unused,
1920                       src0_sel:$src0_sel),
1921                  // VOP1_SDWA with omod
1922                  (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
1923                       clampmod:$clamp, omod:$omod,
1924                       dst_sel:$dst_sel, dst_unused:$dst_unused,
1925                       src0_sel:$src0_sel)),
1926            !if(!eq(NumSrcArgs, 2),
1927               !if(!eq(DstVT.Size, 1),
1928                  // VOPC_SDWA
1929                  (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
1930                       Src1Mod:$src1_modifiers, Src1RC:$src1,
1931                       clampmod:$clamp, src0_sel:$src0_sel, src1_sel:$src1_sel),
1932                  // VOP2_SDWA
1933                  !if(!not(HasSDWAOMod),
1934                     // VOP2_SDWA without omod
1935                     (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
1936                          Src1Mod:$src1_modifiers, Src1RC:$src1,
1937                          clampmod:$clamp,
1938                          dst_sel:$dst_sel, dst_unused:$dst_unused,
1939                          src0_sel:$src0_sel, src1_sel:$src1_sel),
1940                     // VOP2_SDWA with omod
1941                     (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
1942                          Src1Mod:$src1_modifiers, Src1RC:$src1,
1943                          clampmod:$clamp, omod:$omod,
1944                          dst_sel:$dst_sel, dst_unused:$dst_unused,
1945                          src0_sel:$src0_sel, src1_sel:$src1_sel))),
1946            (ins)/* endif */)));
1947}
1948
1949// Outs for DPP
1950class getOutsDPP <bit HasDst, ValueType DstVT, RegisterOperand DstRCDPP> {
1951  dag ret = !if(HasDst,
1952                !if(!eq(DstVT.Size, 1),
1953                    (outs), // no dst for VOPC, we use "vcc"-token as dst in SDWA VOPC instructions
1954                    (outs DstRCDPP:$vdst)),
1955                (outs)); // V_NOP
1956}
1957
1958// Outs for SDWA
1959class getOutsSDWA <bit HasDst, ValueType DstVT, RegisterOperand DstRCSDWA> {
1960  dag ret = !if(HasDst,
1961                !if(!eq(DstVT.Size, 1),
1962                    (outs DstRCSDWA:$sdst),
1963                    (outs DstRCSDWA:$vdst)),
1964                (outs)); // V_NOP
1965}
1966
1967// Returns the assembly string for the inputs and outputs of a VOP[12C]
1968// instruction.  This does not add the _e32 suffix, so it can be reused
1969// by getAsm64.
1970class getAsm32 <bit HasDst, int NumSrcArgs, ValueType DstVT = i32> {
1971  string dst = !if(!eq(DstVT.Size, 1), "$sdst", "$vdst"); // use $sdst for VOPC
1972  string src0 = ", $src0";
1973  string src1 = ", $src1";
1974  string src2 = ", $src2";
1975  string ret = !if(HasDst, dst, "") #
1976               !if(!eq(NumSrcArgs, 1), src0, "") #
1977               !if(!eq(NumSrcArgs, 2), src0#src1, "") #
1978               !if(!eq(NumSrcArgs, 3), src0#src1#src2, "");
1979}
1980
1981// Returns the assembly string for the inputs and outputs of a VOP3
1982// instruction.
1983class getAsm64 <bit HasDst, int NumSrcArgs, bit HasIntClamp, bit HasModifiers,
1984                bit HasOMod, ValueType DstVT = i32> {
1985  string dst = !if(!eq(DstVT.Size, 1), "$sdst", "$vdst"); // use $sdst for VOPC
1986  string src0 = !if(!eq(NumSrcArgs, 1), "$src0_modifiers", "$src0_modifiers,");
1987  string src1 = !if(!eq(NumSrcArgs, 1), "",
1988                   !if(!eq(NumSrcArgs, 2), " $src1_modifiers",
1989                                           " $src1_modifiers,"));
1990  string src2 = !if(!eq(NumSrcArgs, 3), " $src2_modifiers", "");
1991  string iclamp = !if(HasIntClamp, "$clamp", "");
1992  string ret =
1993  !if(!not(HasModifiers),
1994      getAsm32<HasDst, NumSrcArgs, DstVT>.ret # iclamp,
1995      dst#", "#src0#src1#src2#"$clamp"#!if(HasOMod, "$omod", ""));
1996}
1997
1998// Returns the assembly string for the inputs and outputs of a VOP3P
1999// instruction.
2000class getAsmVOP3P <int NumSrcArgs, bit HasModifiers,
2001                   bit HasClamp> {
2002  string dst = "$vdst";
2003  string src0 = !if(!eq(NumSrcArgs, 1), "$src0", "$src0,");
2004  string src1 = !if(!eq(NumSrcArgs, 1), "",
2005                   !if(!eq(NumSrcArgs, 2), " $src1",
2006                                           " $src1,"));
2007  string src2 = !if(!eq(NumSrcArgs, 3), " $src2", "");
2008
2009  string mods = !if(HasModifiers, "$neg_lo$neg_hi", "");
2010  string clamp = !if(HasClamp, "$clamp", "");
2011
2012  // Each modifier is printed as an array of bits for each operand, so
2013  // all operands are printed as part of src0_modifiers.
2014  string ret = dst#", "#src0#src1#src2#"$op_sel$op_sel_hi"#mods#clamp;
2015}
2016
2017class getAsmVOP3OpSel <int NumSrcArgs,
2018                       bit HasClamp,
2019                       bit Src0HasMods,
2020                       bit Src1HasMods,
2021                       bit Src2HasMods> {
2022  string dst = "$vdst";
2023
2024  string isrc0 = !if(!eq(NumSrcArgs, 1), "$src0", "$src0,");
2025  string isrc1 = !if(!eq(NumSrcArgs, 1), "",
2026                     !if(!eq(NumSrcArgs, 2), " $src1",
2027                                             " $src1,"));
2028  string isrc2 = !if(!eq(NumSrcArgs, 3), " $src2", "");
2029
2030  string fsrc0 = !if(!eq(NumSrcArgs, 1), "$src0_modifiers", "$src0_modifiers,");
2031  string fsrc1 = !if(!eq(NumSrcArgs, 1), "",
2032                     !if(!eq(NumSrcArgs, 2), " $src1_modifiers",
2033                                             " $src1_modifiers,"));
2034  string fsrc2 = !if(!eq(NumSrcArgs, 3), " $src2_modifiers", "");
2035
2036  string src0 = !if(Src0HasMods, fsrc0, isrc0);
2037  string src1 = !if(Src1HasMods, fsrc1, isrc1);
2038  string src2 = !if(Src2HasMods, fsrc2, isrc2);
2039
2040  string clamp = !if(HasClamp, "$clamp", "");
2041
2042  string ret = dst#", "#src0#src1#src2#"$op_sel"#clamp;
2043}
2044
2045class getAsmDPP <bit HasDst, int NumSrcArgs, bit HasModifiers, ValueType DstVT = i32> {
2046  string dst = !if(HasDst,
2047                   !if(!eq(DstVT.Size, 1),
2048                       "$sdst",
2049                       "$vdst"),
2050                    ""); // use $sdst for VOPC
2051  string src0 = !if(!eq(NumSrcArgs, 1), "$src0_modifiers", "$src0_modifiers,");
2052  string src1 = !if(!eq(NumSrcArgs, 1), "",
2053                   !if(!eq(NumSrcArgs, 2), " $src1_modifiers",
2054                                           " $src1_modifiers,"));
2055  string args = !if(!not(HasModifiers),
2056                     getAsm32<0, NumSrcArgs, DstVT>.ret,
2057                     ", "#src0#src1);
2058  string ret = dst#args#" $dpp_ctrl$row_mask$bank_mask$bound_ctrl";
2059}
2060
2061class getAsmDPP16 <bit HasDst, int NumSrcArgs, bit HasModifiers, ValueType DstVT = i32> {
2062  string ret = getAsmDPP<HasDst, NumSrcArgs, HasModifiers, DstVT>.ret#"$fi";
2063}
2064
2065class getAsmDPP8 <bit HasDst, int NumSrcArgs, bit HasModifiers, ValueType DstVT = i32>
2066  : getAsmDPP<HasDst, NumSrcArgs, HasModifiers, DstVT> {
2067  let ret = dst#args#" $dpp8$fi";
2068}
2069
2070
2071class getAsmSDWA <bit HasDst, int NumSrcArgs, ValueType DstVT = i32> {
2072  string dst = !if(HasDst,
2073                   !if(!eq(DstVT.Size, 1),
2074                       " vcc", // use vcc token as dst for VOPC instructions
2075                       "$vdst"),
2076                    "");
2077  string src0 = "$src0_modifiers";
2078  string src1 = "$src1_modifiers";
2079  string args = !if(!eq(NumSrcArgs, 0),
2080                    "",
2081                    !if(!eq(NumSrcArgs, 1),
2082                        ", "#src0#"$clamp",
2083                        ", "#src0#", "#src1#"$clamp"
2084                     )
2085                );
2086  string sdwa = !if(!eq(NumSrcArgs, 0),
2087                    "",
2088                    !if(!eq(NumSrcArgs, 1),
2089                        " $dst_sel $dst_unused $src0_sel",
2090                        !if(!eq(DstVT.Size, 1),
2091                            " $src0_sel $src1_sel", // No dst_sel and dst_unused for VOPC
2092                            " $dst_sel $dst_unused $src0_sel $src1_sel"
2093                        )
2094                    )
2095                );
2096  string ret = dst#args#sdwa;
2097}
2098
2099class getAsmSDWA9 <bit HasDst, bit HasOMod, int NumSrcArgs,
2100                   ValueType DstVT = i32> {
2101  string dst = !if(HasDst,
2102                   !if(!eq(DstVT.Size, 1),
2103                       "$sdst", // VOPC
2104                       "$vdst"), // VOP1/2
2105                    "");
2106  string src0 = "$src0_modifiers";
2107  string src1 = "$src1_modifiers";
2108  string out_mods = !if(!not(HasOMod), "$clamp", "$clamp$omod");
2109  string args = !if(!eq(NumSrcArgs, 0), "",
2110                    !if(!eq(NumSrcArgs, 1),
2111                        ", "#src0,
2112                        ", "#src0#", "#src1
2113                     )
2114                );
2115  string sdwa = !if(!eq(NumSrcArgs, 0), "",
2116                    !if(!eq(NumSrcArgs, 1),
2117                        out_mods#" $dst_sel $dst_unused $src0_sel",
2118                        !if(!eq(DstVT.Size, 1),
2119                            " $src0_sel $src1_sel", // No dst_sel, dst_unused and output modifiers for VOPC
2120                            out_mods#" $dst_sel $dst_unused $src0_sel $src1_sel"
2121                        )
2122                    )
2123                );
2124  string ret = dst#args#sdwa;
2125}
2126
2127class getHas64BitOps <int NumSrcArgs, ValueType DstVT, ValueType Src0VT,
2128                      ValueType Src1VT> {
2129  bit ret = !if(!eq(NumSrcArgs, 3),
2130                0,
2131                !if(!eq(DstVT.Size, 64),
2132                    1,
2133                    !if(!eq(Src0VT.Size, 64),
2134                        1,
2135                        !if(!eq(Src1VT.Size, 64),
2136                            1,
2137                            0
2138                        )
2139                    )
2140                )
2141            );
2142}
2143
2144class getHasSDWA <int NumSrcArgs, ValueType DstVT = i32, ValueType Src0VT = i32,
2145                  ValueType Src1VT = i32> {
2146  bit ret = !if(!eq(NumSrcArgs, 3),
2147                0, // NumSrcArgs == 3 - No SDWA for VOP3
2148                !if(!eq(DstVT.Size, 64),
2149                    0, // 64-bit dst - No SDWA for 64-bit operands
2150                    !if(!eq(Src0VT.Size, 64),
2151                        0, // 64-bit src0
2152                        !if(!eq(Src1VT.Size, 64),
2153                            0, // 64-bit src2
2154                            1
2155                        )
2156                    )
2157                )
2158            );
2159}
2160
2161class getHasDPP <int NumSrcArgs> {
2162  bit ret = !if(!eq(NumSrcArgs, 3),
2163                0, // NumSrcArgs == 3 - No DPP for VOP3
2164                1);
2165}
2166
2167class getHasExt32BitDPP <int NumSrcArgs, ValueType DstVT = i32, ValueType Src0VT = i32,
2168                 ValueType Src1VT = i32> {
2169  bit ret = !and(getHasDPP<NumSrcArgs>.ret,
2170                 !not(getHas64BitOps<NumSrcArgs, DstVT, Src0VT, Src1VT>.ret));
2171}
2172
2173class getHasExt64BitDPP <int NumSrcArgs, ValueType DstVT = i32, ValueType Src0VT = i32,
2174                 ValueType Src1VT = i32> {
2175  bit ret = !and(getHasDPP<NumSrcArgs>.ret,
2176                 getHas64BitOps<NumSrcArgs, DstVT, Src0VT, Src1VT>.ret);
2177}
2178
2179// Function that checks if instruction supports DPP and SDWA
2180class getHasExt <int NumSrcArgs, ValueType DstVT = i32, ValueType Src0VT = i32,
2181                 ValueType Src1VT = i32> {
2182  bit ret = !or(getHasDPP<NumSrcArgs>.ret,
2183                getHasSDWA<NumSrcArgs, DstVT, Src0VT, Src1VT>.ret);
2184}
2185
2186// Return an AGPR+VGPR operand class for the given VGPR register class.
2187class getLdStRegisterOperand<RegisterClass RC> {
2188  RegisterOperand ret =
2189    !if(!eq(RC.Size, 32), AVLdSt_32,
2190      !if(!eq(RC.Size, 64), AVLdSt_64,
2191        !if(!eq(RC.Size, 96), AVLdSt_96,
2192          !if(!eq(RC.Size, 128), AVLdSt_128,
2193            !if(!eq(RC.Size, 160), AVLdSt_160,
2194              RegisterOperand<VReg_1> // invalid register
2195    )))));
2196}
2197
2198class BitOr<bit a, bit b> {
2199  bit ret = !if(a, 1, !if(b, 1, 0));
2200}
2201
2202class BitAnd<bit a, bit b> {
2203  bit ret = !if(a, !if(b, 1, 0), 0);
2204}
2205
2206def PatGenMode {
2207  int NoPattern = 0;
2208  int Pattern   = 1;
2209}
2210
2211class VOPProfile <list<ValueType> _ArgVT, bit _EnableF32SrcMods = 0,
2212                  bit _EnableClamp = 0> {
2213
2214  field list<ValueType> ArgVT = _ArgVT;
2215  field bit EnableF32SrcMods = _EnableF32SrcMods;
2216  field bit EnableClamp = _EnableClamp;
2217
2218  field ValueType DstVT = ArgVT[0];
2219  field ValueType Src0VT = ArgVT[1];
2220  field ValueType Src1VT = ArgVT[2];
2221  field ValueType Src2VT = ArgVT[3];
2222  field RegisterOperand DstRC = getVALUDstForVT<DstVT>.ret;
2223  field RegisterOperand DstRCDPP = getVALUDstForVT<DstVT>.ret;
2224  field RegisterOperand DstRCSDWA = getSDWADstForVT<DstVT>.ret;
2225  field RegisterOperand Src0RC32 = getVOPSrc0ForVT<Src0VT>.ret;
2226  field RegisterClass Src1RC32 = getVregSrcForVT<Src1VT>.ret;
2227  field RegisterOperand Src0RC64 = getVOP3SrcForVT<Src0VT>.ret;
2228  field RegisterOperand Src1RC64 = getVOP3SrcForVT<Src1VT>.ret;
2229  field RegisterOperand Src2RC64 = getVOP3SrcForVT<Src2VT>.ret;
2230  field RegisterClass Src0DPP = getVregSrcForVT<Src0VT>.ret;
2231  field RegisterClass Src1DPP = getVregSrcForVT<Src1VT>.ret;
2232  field RegisterOperand Src0SDWA = getSDWASrcForVT<Src0VT>.ret;
2233  field RegisterOperand Src1SDWA = getSDWASrcForVT<Src0VT>.ret;
2234  field Operand Src0Mod = getSrcMod<Src0VT, EnableF32SrcMods>.ret;
2235  field Operand Src1Mod = getSrcMod<Src1VT, EnableF32SrcMods>.ret;
2236  field Operand Src2Mod = getSrcMod<Src2VT, EnableF32SrcMods>.ret;
2237  field Operand Src0ModDPP = getSrcModDPP<Src0VT>.ret;
2238  field Operand Src1ModDPP = getSrcModDPP<Src1VT>.ret;
2239  field Operand Src0ModSDWA = getSrcModSDWA<Src0VT>.ret;
2240  field Operand Src1ModSDWA = getSrcModSDWA<Src1VT>.ret;
2241
2242
2243  field bit HasDst = !ne(DstVT.Value, untyped.Value);
2244  field bit HasDst32 = HasDst;
2245  field bit EmitDst = HasDst; // force dst encoding, see v_movreld_b32 special case
2246  field bit EmitDstSel = EmitDst;
2247  field int NumSrcArgs = getNumSrcArgs<Src0VT, Src1VT, Src2VT>.ret;
2248  field bit HasSrc0 = !ne(Src0VT.Value, untyped.Value);
2249  field bit HasSrc1 = !ne(Src1VT.Value, untyped.Value);
2250  field bit HasSrc2 = !ne(Src2VT.Value, untyped.Value);
2251
2252  // HasSrc*FloatMods affects the SDWA encoding. We ignore EnableF32SrcMods.
2253  field bit HasSrc0FloatMods = isFloatType<Src0VT>.ret;
2254  field bit HasSrc1FloatMods = isFloatType<Src1VT>.ret;
2255  field bit HasSrc2FloatMods = isFloatType<Src2VT>.ret;
2256
2257  // HasSrc*IntMods affects the SDWA encoding. We ignore EnableF32SrcMods.
2258  field bit HasSrc0IntMods = isIntType<Src0VT>.ret;
2259  field bit HasSrc1IntMods = isIntType<Src1VT>.ret;
2260  field bit HasSrc2IntMods = isIntType<Src2VT>.ret;
2261
2262  field bit HasClamp = !or(isModifierType<Src0VT>.ret, EnableClamp);
2263  field bit HasSDWAClamp = EmitDst;
2264  field bit HasFPClamp = !and(isFloatType<DstVT>.ret, HasClamp);
2265  field bit HasIntClamp = !if(isFloatType<DstVT>.ret, 0, HasClamp);
2266  field bit HasClampLo = HasClamp;
2267  field bit HasClampHi = !and(isPackedType<DstVT>.ret, HasClamp);
2268  field bit HasHigh = 0;
2269
2270  field bit IsPacked = isPackedType<Src0VT>.ret;
2271  field bit HasOpSel = IsPacked;
2272  field bit HasOMod = !if(HasOpSel, 0, isFloatType<DstVT>.ret);
2273  field bit HasSDWAOMod = isFloatType<DstVT>.ret;
2274
2275  field bit HasModifiers = !or(isModifierType<Src0VT>.ret,
2276                               isModifierType<Src1VT>.ret,
2277                               isModifierType<Src2VT>.ret,
2278                               HasOMod,
2279                               EnableF32SrcMods);
2280
2281  field bit HasSrc0Mods = HasModifiers;
2282  field bit HasSrc1Mods = !if(HasModifiers, !or(HasSrc1FloatMods, HasSrc1IntMods), 0);
2283  field bit HasSrc2Mods = !if(HasModifiers, !or(HasSrc2FloatMods, HasSrc2IntMods), 0);
2284
2285  field bit HasExt = getHasExt<NumSrcArgs, DstVT, Src0VT, Src1VT>.ret;
2286  field bit HasExtDPP = getHasDPP<NumSrcArgs>.ret;
2287  field bit HasExt32BitDPP = getHasExt32BitDPP<NumSrcArgs, DstVT, Src0VT, Src1VT>.ret;
2288  field bit HasExt64BitDPP = getHasExt64BitDPP<NumSrcArgs, DstVT, Src0VT, Src1VT>.ret;
2289  field bit HasExtSDWA = getHasSDWA<NumSrcArgs, DstVT, Src0VT, Src1VT>.ret;
2290  field bit HasExtSDWA9 = HasExtSDWA;
2291  field int NeedPatGen = PatGenMode.NoPattern;
2292
2293  field bit IsMAI = 0;
2294  field bit IsDOT = 0;
2295  field bit IsSingle = 0;
2296
2297  field Operand Src0PackedMod = !if(HasSrc0FloatMods, PackedF16InputMods, PackedI16InputMods);
2298  field Operand Src1PackedMod = !if(HasSrc1FloatMods, PackedF16InputMods, PackedI16InputMods);
2299  field Operand Src2PackedMod = !if(HasSrc2FloatMods, PackedF16InputMods, PackedI16InputMods);
2300
2301  field dag Outs = !if(HasDst,(outs DstRC:$vdst),(outs));
2302
2303  // VOP3b instructions are a special case with a second explicit
2304  // output. This is manually overridden for them.
2305  field dag Outs32 = Outs;
2306  field dag Outs64 = Outs;
2307  field dag OutsDPP = getOutsDPP<HasDst, DstVT, DstRCDPP>.ret;
2308  field dag OutsDPP8 = getOutsDPP<HasDst, DstVT, DstRCDPP>.ret;
2309  field dag OutsSDWA = getOutsSDWA<HasDst, DstVT, DstRCSDWA>.ret;
2310
2311  field dag Ins32 = getIns32<Src0RC32, Src1RC32, NumSrcArgs>.ret;
2312  field dag Ins64 = getIns64<Src0RC64, Src1RC64, Src2RC64, NumSrcArgs,
2313                             HasIntClamp, HasModifiers, HasSrc2Mods,
2314                             HasOMod, Src0Mod, Src1Mod, Src2Mod>.ret;
2315  field dag InsVOP3P = getInsVOP3P<Src0RC64, Src1RC64, Src2RC64,
2316                                   NumSrcArgs, HasClamp,
2317                                   Src0PackedMod, Src1PackedMod, Src2PackedMod>.ret;
2318  field dag InsVOP3OpSel = getInsVOP3OpSel<Src0RC64, Src1RC64, Src2RC64,
2319                                NumSrcArgs, HasClamp, HasOMod,
2320                                getOpSelMod<Src0VT>.ret,
2321                                getOpSelMod<Src1VT>.ret,
2322                                getOpSelMod<Src2VT>.ret>.ret;
2323  field dag InsDPP = !if(HasExtDPP,
2324                         getInsDPP<DstRCDPP, Src0DPP, Src1DPP, NumSrcArgs,
2325                                   HasModifiers, Src0ModDPP, Src1ModDPP>.ret,
2326                         (ins));
2327  field dag InsDPP16 = getInsDPP16<DstRCDPP, Src0DPP, Src1DPP, NumSrcArgs,
2328                                   HasModifiers, Src0ModDPP, Src1ModDPP>.ret;
2329  field dag InsDPP8 = getInsDPP8<DstRCDPP, Src0DPP, Src1DPP, NumSrcArgs, 0,
2330                                 Src0ModDPP, Src1ModDPP>.ret;
2331  field dag InsSDWA = getInsSDWA<Src0SDWA, Src1SDWA, NumSrcArgs,
2332                                 HasSDWAOMod, Src0ModSDWA, Src1ModSDWA,
2333                                 DstVT>.ret;
2334
2335
2336  field string Asm32 = getAsm32<HasDst, NumSrcArgs, DstVT>.ret;
2337  field string Asm64 = getAsm64<HasDst, NumSrcArgs, HasIntClamp, HasModifiers, HasOMod, DstVT>.ret;
2338  field string AsmVOP3P = getAsmVOP3P<NumSrcArgs, HasModifiers, HasClamp>.ret;
2339  field string AsmVOP3OpSel = getAsmVOP3OpSel<NumSrcArgs,
2340                                              HasClamp,
2341                                              HasSrc0FloatMods,
2342                                              HasSrc1FloatMods,
2343                                              HasSrc2FloatMods>.ret;
2344  field string AsmDPP = !if(HasExtDPP,
2345                            getAsmDPP<HasDst, NumSrcArgs, HasModifiers, DstVT>.ret, "");
2346  field string AsmDPP16 = getAsmDPP16<HasDst, NumSrcArgs, HasModifiers, DstVT>.ret;
2347  // DPP8 encoding has no fields for modifiers, and it is enforced by setting
2348  // the asm operand name via this HasModifiers flag
2349  field string AsmDPP8 = getAsmDPP8<HasDst, NumSrcArgs, 0 /*HasModifiers*/, DstVT>.ret;
2350  field string AsmSDWA = getAsmSDWA<HasDst, NumSrcArgs, DstVT>.ret;
2351  field string AsmSDWA9 = getAsmSDWA9<HasDst, HasSDWAOMod, NumSrcArgs, DstVT>.ret;
2352
2353  field string TieRegDPP = "$old";
2354}
2355
2356class VOP_NO_EXT <VOPProfile p> : VOPProfile <p.ArgVT> {
2357  let HasExt = 0;
2358  let HasExtDPP = 0;
2359  let HasExt32BitDPP = 0;
2360  let HasExt64BitDPP = 0;
2361  let HasExtSDWA = 0;
2362  let HasExtSDWA9 = 0;
2363}
2364
2365class VOP_PAT_GEN <VOPProfile p, int mode=PatGenMode.NoPattern> : VOPProfile <p.ArgVT> {
2366  let NeedPatGen = mode;
2367}
2368
2369def VOP_F16_F16 : VOPProfile <[f16, f16, untyped, untyped]>;
2370def VOP_F16_I16 : VOPProfile <[f16, i16, untyped, untyped]>;
2371def VOP_I16_F16 : VOPProfile <[i16, f16, untyped, untyped]>;
2372
2373def VOP_F16_F16_F16 : VOPProfile <[f16, f16, f16, untyped]>;
2374def VOP_F16_F16_I16 : VOPProfile <[f16, f16, i16, untyped]>;
2375def VOP_F16_F16_I32 : VOPProfile <[f16, f16, i32, untyped]>;
2376def VOP_I16_I16_I16 : VOPProfile <[i16, i16, i16, untyped]>;
2377def VOP_I16_I16_I16_ARITH : VOPProfile <[i16, i16, i16, untyped], 0, /*EnableClamp=*/1>;
2378
2379def VOP_I16_I16_I16_I16 : VOPProfile <[i16, i16, i16, i16, untyped]>;
2380def VOP_F16_F16_F16_F16 : VOPProfile <[f16, f16, f16, f16, untyped]>;
2381
2382def VOP_I32_I16_I16_I32 : VOPProfile <[i32, i16, i16, i32, untyped]>;
2383
2384def VOP_V2F16_V2F16_V2F16 : VOPProfile <[v2f16, v2f16, v2f16, untyped]>;
2385def VOP_V2I16_V2I16_V2I16 : VOPProfile <[v2i16, v2i16, v2i16, untyped]>;
2386def VOP_B32_F16_F16 : VOPProfile <[i32, f16, f16, untyped]>;
2387
2388def VOP_V2F16_V2F16_V2F16_V2F16 : VOPProfile <[v2f16, v2f16, v2f16, v2f16]>;
2389def VOP_V2I16_V2I16_V2I16_V2I16 : VOPProfile <[v2i16, v2i16, v2i16, v2i16]>;
2390def VOP_V2I16_F32_F32 : VOPProfile <[v2i16, f32, f32, untyped]>;
2391def VOP_V2I16_I32_I32 : VOPProfile <[v2i16, i32, i32, untyped]>;
2392
2393def VOP_F32_V2F16_V2F16_V2F16 : VOPProfile <[f32, v2f16, v2f16, v2f16]>;
2394
2395def VOP_NONE : VOPProfile <[untyped, untyped, untyped, untyped]>;
2396
2397def VOP_F32_F32 : VOPProfile <[f32, f32, untyped, untyped]>;
2398def VOP_F32_F64 : VOPProfile <[f32, f64, untyped, untyped]>;
2399def VOP_F32_I32 : VOPProfile <[f32, i32, untyped, untyped]>;
2400def VOP_F64_F32 : VOPProfile <[f64, f32, untyped, untyped]>;
2401def VOP_F64_F64 : VOPProfile <[f64, f64, untyped, untyped]>;
2402def VOP_F64_I32 : VOPProfile <[f64, i32, untyped, untyped]>;
2403def VOP_I32_F32 : VOPProfile <[i32, f32, untyped, untyped]>;
2404def VOP_I32_F64 : VOPProfile <[i32, f64, untyped, untyped]>;
2405def VOP_I32_I32 : VOPProfile <[i32, i32, untyped, untyped]>;
2406def VOP_F16_F32 : VOPProfile <[f16, f32, untyped, untyped]>;
2407def VOP_F32_F16 : VOPProfile <[f32, f16, untyped, untyped]>;
2408def VOP_I64_I64 : VOPProfile <[i64, i64, untyped, untyped]>;
2409
2410def VOP_F32_F32_F16 : VOPProfile <[f32, f32, f16, untyped]>;
2411def VOP_F32_F32_F32 : VOPProfile <[f32, f32, f32, untyped]>;
2412def VOP_F32_F32_I32 : VOPProfile <[f32, f32, i32, untyped]>;
2413def VOP_F64_F64_F64 : VOPProfile <[f64, f64, f64, untyped]>;
2414def VOP_F64_F64_I32 : VOPProfile <[f64, f64, i32, untyped]>;
2415def VOP_I32_F32_F32 : VOPProfile <[i32, f32, f32, untyped]>;
2416def VOP_I32_F32_I32 : VOPProfile <[i32, f32, i32, untyped]>;
2417def VOP_I32_I32_I32 : VOPProfile <[i32, i32, i32, untyped]>;
2418def VOP_I32_I32_I32_ARITH : VOPProfile <[i32, i32, i32, untyped], 0, /*EnableClamp=*/1>;
2419def VOP_V2F16_F32_F32 : VOPProfile <[v2f16, f32, f32, untyped]>;
2420def VOP_F32_F16_F16_F16 : VOPProfile <[f32, f16, f16, f16]>;
2421
2422def VOP_I64_I64_I32 : VOPProfile <[i64, i64, i32, untyped]>;
2423def VOP_I64_I32_I64 : VOPProfile <[i64, i32, i64, untyped]>;
2424def VOP_I64_I64_I64 : VOPProfile <[i64, i64, i64, untyped]>;
2425
2426def VOP_F16_F32_F16_F32 : VOPProfile <[f16, f32, f16, f32]>;
2427def VOP_F32_F32_F16_F16 : VOPProfile <[f32, f32, f16, f16]>;
2428def VOP_F32_F32_F32_F32 : VOPProfile <[f32, f32, f32, f32]>;
2429def VOP_F64_F64_F64_F64 : VOPProfile <[f64, f64, f64, f64]>;
2430def VOP_I32_I32_I32_I32 : VOPProfile <[i32, i32, i32, i32]>;
2431def VOP_I64_I32_I32_I64 : VOPProfile <[i64, i32, i32, i64]>;
2432def VOP_I32_F32_I32_I32 : VOPProfile <[i32, f32, i32, i32]>;
2433def VOP_I64_I64_I32_I64 : VOPProfile <[i64, i64, i32, i64]>;
2434def VOP_V4I32_I64_I32_V4I32 : VOPProfile <[v4i32, i64, i32, v4i32]>;
2435
2436def VOP_F32_V2F16_V2F16_F32 : VOPProfile <[f32, v2f16, v2f16, f32]>;
2437def VOP_I32_V2I16_V2I16_I32 : VOPProfile <[i32, v2i16, v2i16, i32]>;
2438
2439def VOP_V4F32_F32_F32_V4F32       : VOPProfile <[v4f32,  f32,   f32,   v4f32]>;
2440def VOP_V16F32_F32_F32_V16F32     : VOPProfile <[v16f32, f32,   f32,   v16f32]>;
2441def VOP_V32F32_F32_F32_V32F32     : VOPProfile <[v32f32, f32,   f32,   v32f32]>;
2442def VOP_V4F32_V4F16_V4F16_V4F32   : VOPProfile <[v4f32,  v4f16, v4f16, v4f32]>;
2443def VOP_V16F32_V4F16_V4F16_V16F32 : VOPProfile <[v16f32, v4f16, v4f16, v16f32]>;
2444def VOP_V32F32_V4F16_V4F16_V32F32 : VOPProfile <[v32f32, v4f16, v4f16, v32f32]>;
2445def VOP_V4F32_V2I16_V2I16_V4F32   : VOPProfile <[v4f32,  v2i16, v2i16, v4f32]>;
2446def VOP_V16F32_V2I16_V2I16_V16F32 : VOPProfile <[v16f32, v2i16, v2i16, v16f32]>;
2447def VOP_V32F32_V2I16_V2I16_V32F32 : VOPProfile <[v32f32, v2i16, v2i16, v32f32]>;
2448def VOP_V4I32_I32_I32_V4I32       : VOPProfile <[v4i32,  i32,   i32,   v4i32]>;
2449def VOP_V16I32_I32_I32_V16I32     : VOPProfile <[v16i32, i32,   i32,   v16i32]>;
2450def VOP_V32I32_I32_I32_V32I32     : VOPProfile <[v32i32, i32,   i32,   v32i32]>;
2451
2452def VOP_V4F64_F64_F64_V4F64 : VOPProfile <[v4f64, f64, f64, v4f64]>;
2453def VOP_V1F64_F64_F64_V1F64 : VOPProfile <[v1f64, f64, f64, v1f64]>;
2454
2455def VOP_V2F32_V2F32_V2F32_V2F32   : VOPProfile <[v2f32,  v2f32, v2f32, v2f32]>;
2456def VOP_V2F32_V2F32_V2F32         : VOPProfile <[v2f32,  v2f32, v2f32, untyped]>;
2457def VOP_V2I32_V2I32_V2I32         : VOPProfile <[v2i32,  v2i32, v2i32, untyped]>;
2458def VOP_V4F32_V4I16_V4I16_V4F32   : VOPProfile <[v4f32,  v4i16, v4i16, v4f32]>;
2459def VOP_V16F32_V4I16_V4I16_V16F32 : VOPProfile <[v16f32, v4i16, v4i16, v16f32]>;
2460def VOP_V32F32_V4I16_V4I16_V32F32 : VOPProfile <[v32f32, v4i16, v4i16, v32f32]>;
2461
2462def VOP_V4I32_I64_I64_V4I32       : VOPProfile <[v4i32,  i64,   i64,   v4i32]>;
2463def VOP_V16I32_I64_I64_V16I32     : VOPProfile <[v16i32, i64,   i64,   v16i32]>;
2464def VOP_V4F32_V2F32_V2F32_V4F32   : VOPProfile <[v4f32,  v2f32, v2f32, v4f32]>;
2465def VOP_V16F32_V2F32_V2F32_V16F32 : VOPProfile <[v16f32, v2f32, v2f32, v16f32]>;
2466
2467def VOP_V4F32_V4F16_V8F16_I32     : VOPProfile <[v4f32,  v4f16, v8f16, i32]>;
2468def VOP_V16F32_V4F16_V8F16_I32    : VOPProfile <[v16f32, v4f16, v8f16, i32]>;
2469def VOP_V4F32_V4I16_V8I16_I32     : VOPProfile <[v4f32,  v4i16, v8i16, i32]>;
2470def VOP_V16F32_V4I16_V8I16_I32    : VOPProfile <[v16f32, v4i16, v8i16, i32]>;
2471def VOP_V4I32_V2I32_V4I32_I32     : VOPProfile <[v4i32,  v2i32, v4i32, i32]>;
2472def VOP_V16I32_V2I32_V4I32_I32    : VOPProfile <[v16i32, v2i32, v4i32, i32]>;
2473
2474class Commutable_REV <string revOp, bit isOrig> {
2475  string RevOp = revOp;
2476  bit IsOrig = isOrig;
2477}
2478
2479class AtomicNoRet <string noRetOp, bit isRet> {
2480  string NoRetOp = noRetOp;
2481  bit IsRet = isRet;
2482}
2483
2484//===----------------------------------------------------------------------===//
2485// Interpolation opcodes
2486//===----------------------------------------------------------------------===//
2487
2488class VINTRPDstOperand <RegisterClass rc> : RegisterOperand <rc, "printVINTRPDst">;
2489
2490class VINTRP_Pseudo <string opName, dag outs, dag ins, list<dag> pattern> :
2491  VINTRPCommon <outs, ins, "", pattern>,
2492  SIMCInstr<opName, SIEncodingFamily.NONE> {
2493  let isPseudo = 1;
2494  let isCodeGenOnly = 1;
2495}
2496
2497// FIXME-GFX10: WIP.
2498class VINTRP_Real_si <bits <2> op, string opName, dag outs, dag ins,
2499                      string asm, int encodingFamily> :
2500  VINTRPCommon <outs, ins, asm, []>,
2501  VINTRPe <op>,
2502  SIMCInstr<opName, encodingFamily> {
2503}
2504
2505class VINTRP_Real_vi <bits <2> op, string opName, dag outs, dag ins,
2506                      string asm> :
2507  VINTRPCommon <outs, ins, asm, []>,
2508  VINTRPe_vi <op>,
2509  SIMCInstr<opName, SIEncodingFamily.VI> {
2510  let AssemblerPredicate = VIAssemblerPredicate;
2511  let DecoderNamespace = "GFX8";
2512}
2513
2514// FIXME-GFX10: WIP.
2515multiclass VINTRP_m <bits <2> op, dag outs, dag ins, string asm,
2516                     list<dag> pattern = []> {
2517  def "" : VINTRP_Pseudo <NAME, outs, ins, pattern>;
2518
2519  let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {
2520    def _si : VINTRP_Real_si <op, NAME, outs, ins, asm, SIEncodingFamily.SI>;
2521  } // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"
2522
2523  def _vi : VINTRP_Real_vi <op, NAME, outs, ins, asm>;
2524
2525  let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in {
2526    def _gfx10 : VINTRP_Real_si<op, NAME, outs, ins, asm, SIEncodingFamily.GFX10>;
2527  } // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10"
2528}
2529
2530//===----------------------------------------------------------------------===//
2531// Vector instruction mappings
2532//===----------------------------------------------------------------------===//
2533
2534// Maps an opcode in e32 form to its e64 equivalent
2535def getVOPe64 : InstrMapping {
2536  let FilterClass = "VOP";
2537  let RowFields = ["OpName"];
2538  let ColFields = ["Size", "VOP3"];
2539  let KeyCol = ["4", "0"];
2540  let ValueCols = [["8", "1"]];
2541}
2542
2543// Maps an opcode in e64 form to its e32 equivalent
2544def getVOPe32 : InstrMapping {
2545  let FilterClass = "VOP";
2546  let RowFields = ["OpName"];
2547  let ColFields = ["Size", "VOP3"];
2548  let KeyCol = ["8", "1"];
2549  let ValueCols = [["4", "0"]];
2550}
2551
2552// Maps ordinary instructions to their SDWA counterparts
2553def getSDWAOp : InstrMapping {
2554  let FilterClass = "VOP";
2555  let RowFields = ["OpName"];
2556  let ColFields = ["AsmVariantName"];
2557  let KeyCol = ["Default"];
2558  let ValueCols = [["SDWA"]];
2559}
2560
2561// Maps SDWA instructions to their ordinary counterparts
2562def getBasicFromSDWAOp : InstrMapping {
2563  let FilterClass = "VOP";
2564  let RowFields = ["OpName"];
2565  let ColFields = ["AsmVariantName"];
2566  let KeyCol = ["SDWA"];
2567  let ValueCols = [["Default"]];
2568}
2569
2570// Maps ordinary instructions to their DPP counterparts
2571def getDPPOp32 : InstrMapping {
2572  let FilterClass = "VOP";
2573  let RowFields = ["OpName"];
2574  let ColFields = ["AsmVariantName"];
2575  let KeyCol = ["Default"];
2576  let ValueCols = [["DPP"]];
2577}
2578
2579// Maps an commuted opcode to its original version
2580def getCommuteOrig : InstrMapping {
2581  let FilterClass = "Commutable_REV";
2582  let RowFields = ["RevOp"];
2583  let ColFields = ["IsOrig"];
2584  let KeyCol = ["0"];
2585  let ValueCols = [["1"]];
2586}
2587
2588// Maps an original opcode to its commuted version
2589def getCommuteRev : InstrMapping {
2590  let FilterClass = "Commutable_REV";
2591  let RowFields = ["RevOp"];
2592  let ColFields = ["IsOrig"];
2593  let KeyCol = ["1"];
2594  let ValueCols = [["0"]];
2595}
2596
2597def getMCOpcodeGen : InstrMapping {
2598  let FilterClass = "SIMCInstr";
2599  let RowFields = ["PseudoInstr"];
2600  let ColFields = ["Subtarget"];
2601  let KeyCol = [!cast<string>(SIEncodingFamily.NONE)];
2602  // These columns must be kept in sync with the SIEncodingFamily enumeration.
2603  let ValueCols = [[!cast<string>(SIEncodingFamily.SI)],
2604                   [!cast<string>(SIEncodingFamily.VI)],
2605                   [!cast<string>(SIEncodingFamily.SDWA)],
2606                   [!cast<string>(SIEncodingFamily.SDWA9)],
2607                   // GFX80 encoding is added to work around a multiple matching
2608                   // issue for buffer instructions with unpacked d16 data. This
2609                   // does not actually change the encoding, and thus may be
2610                   // removed later.
2611                   [!cast<string>(SIEncodingFamily.GFX80)],
2612                   [!cast<string>(SIEncodingFamily.GFX9)],
2613                   [!cast<string>(SIEncodingFamily.GFX10)],
2614                   [!cast<string>(SIEncodingFamily.SDWA10)],
2615                   [!cast<string>(SIEncodingFamily.GFX90A)],
2616                   [!cast<string>(SIEncodingFamily.GFX940)],
2617                   [!cast<string>(SIEncodingFamily.GFX11)]];
2618}
2619
2620// Get equivalent SOPK instruction.
2621def getSOPKOp : InstrMapping {
2622  let FilterClass = "SOPKInstTable";
2623  let RowFields = ["BaseCmpOp"];
2624  let ColFields = ["IsSOPK"];
2625  let KeyCol = ["0"];
2626  let ValueCols = [["1"]];
2627}
2628
2629def getAddr64Inst : InstrMapping {
2630  let FilterClass = "MUBUFAddr64Table";
2631  let RowFields = ["OpName"];
2632  let ColFields = ["IsAddr64"];
2633  let KeyCol = ["0"];
2634  let ValueCols = [["1"]];
2635}
2636
2637def getIfAddr64Inst : InstrMapping {
2638  let FilterClass = "MUBUFAddr64Table";
2639  let RowFields = ["OpName"];
2640  let ColFields = ["IsAddr64"];
2641  let KeyCol = ["1"];
2642  let ValueCols = [["1"]];
2643}
2644
2645// Maps an atomic opcode to its returnless version.
2646def getAtomicNoRetOp : InstrMapping {
2647  let FilterClass = "AtomicNoRet";
2648  let RowFields = ["NoRetOp"];
2649  let ColFields = ["IsRet"];
2650  let KeyCol = ["1"];
2651  let ValueCols = [["0"]];
2652}
2653
2654// Maps a GLOBAL to its SADDR form.
2655def getGlobalSaddrOp : InstrMapping {
2656  let FilterClass = "GlobalSaddrTable";
2657  let RowFields = ["SaddrOp"];
2658  let ColFields = ["IsSaddr"];
2659  let KeyCol = ["0"];
2660  let ValueCols = [["1"]];
2661}
2662
2663// Maps a GLOBAL SADDR to its VADDR form.
2664def getGlobalVaddrOp : InstrMapping {
2665  let FilterClass = "GlobalSaddrTable";
2666  let RowFields = ["SaddrOp"];
2667  let ColFields = ["IsSaddr"];
2668  let KeyCol = ["1"];
2669  let ValueCols = [["0"]];
2670}
2671
2672// Maps a v_cmpx opcode with sdst to opcode without sdst.
2673def getVCMPXNoSDstOp : InstrMapping {
2674  let FilterClass = "VCMPXNoSDstTable";
2675  let RowFields = ["NoSDstOp"];
2676  let ColFields = ["HasSDst"];
2677  let KeyCol = ["1"];
2678  let ValueCols = [["0"]];
2679}
2680
2681// Maps a SOPP to a SOPP with S_NOP
2682def getSOPPWithRelaxation : InstrMapping {
2683  let FilterClass = "SOPPRelaxTable";
2684  let RowFields = ["KeyName"];
2685  let ColFields = ["IsRelaxed"];
2686  let KeyCol = ["0"];
2687  let ValueCols = [["1"]];
2688}
2689
2690// Maps flat scratch opcodes by addressing modes
2691def getFlatScratchInstSTfromSS : InstrMapping {
2692  let FilterClass = "FlatScratchInst";
2693  let RowFields = ["SVOp"];
2694  let ColFields = ["Mode"];
2695  let KeyCol = ["SS"];
2696  let ValueCols = [["ST"]];
2697}
2698
2699def getFlatScratchInstSSfromSV : InstrMapping {
2700  let FilterClass = "FlatScratchInst";
2701  let RowFields = ["SVOp"];
2702  let ColFields = ["Mode"];
2703  let KeyCol = ["SV"];
2704  let ValueCols = [["SS"]];
2705}
2706
2707def getFlatScratchInstSVfromSVS : InstrMapping {
2708  let FilterClass = "FlatScratchInst";
2709  let RowFields = ["SVOp"];
2710  let ColFields = ["Mode"];
2711  let KeyCol = ["SVS"];
2712  let ValueCols = [["SV"]];
2713}
2714
2715def getFlatScratchInstSVfromSS : InstrMapping {
2716  let FilterClass = "FlatScratchInst";
2717  let RowFields = ["SVOp"];
2718  let ColFields = ["Mode"];
2719  let KeyCol = ["SS"];
2720  let ValueCols = [["SV"]];
2721}
2722
2723def getMFMAEarlyClobberOp : InstrMapping {
2724  let FilterClass = "MFMATable";
2725  let RowFields = ["FMAOp"];
2726  let ColFields = ["IsMac"];
2727  let KeyCol = ["1"];
2728  let ValueCols = [["0"]];
2729}
2730
2731// Maps an v_cmp instruction to its v_cmpx equivalent.
2732def getVCMPXOpFromVCMP : InstrMapping {
2733  let FilterClass = "VCMPVCMPXTable";
2734  let RowFields = ["VCMPOp"];
2735  let ColFields = ["IsVCMPX"];
2736  let KeyCol = ["0"];
2737  let ValueCols = [["1"]];
2738}
2739
2740include "SIInstructions.td"
2741
2742include "DSInstructions.td"
2743include "MIMGInstructions.td"
2744