1//===-- DSInstructions.td - DS Instruction Definitions --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9class DS_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> :
10  InstSI <outs, ins, "", pattern>,
11  SIMCInstr <opName, SIEncodingFamily.NONE> {
12
13  let LGKM_CNT = 1;
14  let DS = 1;
15  let Size = 8;
16  let UseNamedOperandTable = 1;
17
18  // Most instruction load and store data, so set this as the default.
19  let mayLoad = 1;
20  let mayStore = 1;
21  let maybeAtomic = 1;
22
23  let hasSideEffects = 0;
24  let SchedRW = [WriteLDS];
25
26  let isPseudo = 1;
27  let isCodeGenOnly = 1;
28
29  let AsmMatchConverter = "cvtDS";
30
31  string Mnemonic = opName;
32  string AsmOperands = asmOps;
33
34  // Well these bits a kind of hack because it would be more natural
35  // to test "outs" and "ins" dags for the presence of particular operands
36  bits<1> has_vdst = 1;
37  bits<1> has_addr = 1;
38  bits<1> has_data0 = 1;
39  bits<1> has_data1 = 1;
40
41  bits<1> has_gws_data0 = 0; // data0 is encoded as addr
42
43  bits<1> has_offset  = 1; // has "offset" that should be split to offset0,1
44  bits<1> has_offset0 = 1;
45  bits<1> has_offset1 = 1;
46
47  bits<1> has_gds = 1;
48  bits<1> gdsValue = 0; // if has_gds == 0 set gds to this value
49
50  bits<1> has_m0_read = 1;
51
52  let Uses = !if(has_m0_read, [M0, EXEC], [EXEC]);
53}
54
55class DS_Real <DS_Pseudo ps, string opName = ps.Mnemonic> :
56  InstSI <ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands>,
57  Enc64 {
58
59  let isPseudo = 0;
60  let isCodeGenOnly = 0;
61  let LGKM_CNT = 1;
62  let DS = 1;
63  let UseNamedOperandTable = 1;
64
65  // copy relevant pseudo op flags
66  let SubtargetPredicate = ps.SubtargetPredicate;
67  let OtherPredicates    = ps.OtherPredicates;
68  let AsmMatchConverter  = ps.AsmMatchConverter;
69  let SchedRW            = ps.SchedRW;
70  let mayLoad            = ps.mayLoad;
71  let mayStore           = ps.mayStore;
72  let IsAtomicRet        = ps.IsAtomicRet;
73  let IsAtomicNoRet      = ps.IsAtomicNoRet;
74
75  let Constraints = ps.Constraints;
76  let DisableEncoding = ps.DisableEncoding;
77
78  // encoding fields
79  bits<10> vdst;
80  bits<1> gds;
81  bits<8> addr;
82  bits<10> data0;
83  bits<10> data1;
84  bits<8> offset0;
85  bits<8> offset1;
86
87  bits<16> offset;
88  let offset0 = !if(ps.has_offset, offset{7-0}, ?);
89  let offset1 = !if(ps.has_offset, offset{15-8}, ?);
90
91  bits<1> acc = !if(ps.has_vdst, vdst{9},
92                    !if(!or(ps.has_data0, ps.has_gws_data0), data0{9}, 0));
93}
94
95// DS Pseudo instructions
96
97class DS_0A1D_NORET<string opName, RegisterClass rc = VGPR_32>
98: DS_Pseudo<opName,
99  (outs),
100  (ins getLdStRegisterOperand<rc>.ret:$data0, offset:$offset, gds:$gds),
101  " $data0$offset$gds"> {
102
103  let has_addr = 0;
104  let has_data1 = 0;
105  let has_vdst = 0;
106}
107
108class DS_1A1D_NORET<string opName, RegisterClass rc = VGPR_32>
109: DS_Pseudo<opName,
110  (outs),
111  (ins VGPR_32:$addr, getLdStRegisterOperand<rc>.ret:$data0, offset:$offset, gds:$gds),
112  " $addr, $data0$offset$gds"> {
113
114  let has_data1 = 0;
115  let has_vdst = 0;
116  let IsAtomicNoRet = 1;
117}
118
119multiclass DS_1A1D_NORET_mc<string opName, RegisterClass rc = VGPR_32> {
120  def "" : DS_1A1D_NORET<opName, rc>,
121           AtomicNoRet<opName, 0>;
122
123  let has_m0_read = 0 in {
124    def _gfx9 : DS_1A1D_NORET<opName, rc>,
125                AtomicNoRet<opName#"_gfx9", 0>;
126  }
127}
128
129multiclass DS_1A1D_NORET_mc_gfx9<string opName, RegisterClass rc = VGPR_32> {
130  let has_m0_read = 0 in {
131    def "" : DS_1A1D_NORET<opName, rc>,
132                AtomicNoRet<opName, 0>;
133  }
134}
135
136class DS_1A2D_NORET<string opName, RegisterClass rc = VGPR_32,
137                    RegisterOperand data_op = getLdStRegisterOperand<rc>.ret>
138: DS_Pseudo<opName,
139  (outs),
140  (ins VGPR_32:$addr, data_op:$data0, data_op:$data1, offset:$offset, gds:$gds),
141  " $addr, $data0, $data1$offset$gds"> {
142
143  let has_vdst = 0;
144  let IsAtomicNoRet = 1;
145}
146
147multiclass DS_1A2D_NORET_mc<string opName, RegisterClass rc = VGPR_32> {
148  def "" : DS_1A2D_NORET<opName, rc>,
149           AtomicNoRet<opName, 0>;
150
151  let has_m0_read = 0 in {
152    def _gfx9 : DS_1A2D_NORET<opName, rc>,
153                AtomicNoRet<opName#"_gfx9", 0>;
154  }
155}
156
157class DS_1A2D_Off8_NORET <string opName, RegisterClass rc = VGPR_32,
158                          RegisterOperand data_op = getLdStRegisterOperand<rc>.ret>
159: DS_Pseudo<opName,
160  (outs),
161  (ins VGPR_32:$addr, data_op:$data0, data_op:$data1,
162       offset0:$offset0, offset1:$offset1, gds:$gds),
163  " $addr, $data0, $data1$offset0$offset1$gds"> {
164
165  let has_vdst = 0;
166  let has_offset = 0;
167  let AsmMatchConverter = "cvtDSOffset01";
168}
169
170multiclass DS_1A2D_Off8_NORET_mc <string opName, RegisterClass rc = VGPR_32> {
171  def "" : DS_1A2D_Off8_NORET<opName, rc>;
172
173  let has_m0_read = 0 in {
174    def _gfx9 : DS_1A2D_Off8_NORET<opName, rc>;
175  }
176}
177
178class DS_0A1D_RET_GDS<string opName, RegisterClass rc = VGPR_32, RegisterClass src = rc,
179                  RegisterOperand dst_op = getLdStRegisterOperand<rc>.ret,
180                  RegisterOperand src_op = getLdStRegisterOperand<src>.ret>
181: DS_Pseudo<opName,
182  (outs dst_op:$vdst),
183  (ins src_op:$data0, offset:$offset),
184  " $vdst, $data0$offset gds"> {
185
186  let has_addr = 0;
187  let has_data1 = 0;
188  let has_gds = 0;
189  let gdsValue = 1;
190  let AsmMatchConverter = "cvtDSGds";
191  let hasSideEffects = 1;
192}
193
194class DS_1A1D_RET <string opName, RegisterClass rc = VGPR_32,
195                  RegisterOperand data_op = getLdStRegisterOperand<rc>.ret>
196: DS_Pseudo<opName,
197  (outs data_op:$vdst),
198  (ins VGPR_32:$addr, data_op:$data0, offset:$offset, gds:$gds),
199  " $vdst, $addr, $data0$offset$gds"> {
200
201  let hasPostISelHook = 1;
202  let has_data1 = 0;
203  let IsAtomicRet = 1;
204}
205
206multiclass DS_1A1D_RET_mc <string opName, RegisterClass rc = VGPR_32,
207                           string NoRetOp = ""> {
208  def "" : DS_1A1D_RET<opName, rc>,
209    AtomicNoRet<NoRetOp, !ne(NoRetOp, "")>;
210
211  let has_m0_read = 0 in {
212    def _gfx9 : DS_1A1D_RET<opName, rc>,
213      AtomicNoRet<!if(!eq(NoRetOp, ""), "", NoRetOp#"_gfx9"),
214                  !ne(NoRetOp, "")>;
215  }
216}
217
218multiclass DS_1A1D_RET_mc_gfx9 <string opName, RegisterClass rc = VGPR_32,
219                                string NoRetOp = ""> {
220  let has_m0_read = 0 in {
221    def "" : DS_1A1D_RET<opName, rc>,
222      AtomicNoRet<!if(!eq(NoRetOp, ""), "", NoRetOp),
223                  !if(!eq(NoRetOp, ""), 0, 1)>;
224  }
225}
226
227class DS_1A2D_RET<string opName,
228                  RegisterClass rc = VGPR_32,
229                  RegisterClass src = rc,
230                  RegisterOperand dst_op = getLdStRegisterOperand<rc>.ret,
231                  RegisterOperand src_op = getLdStRegisterOperand<src>.ret>
232: DS_Pseudo<opName,
233  (outs dst_op:$vdst),
234  (ins VGPR_32:$addr, src_op:$data0, src_op:$data1, offset:$offset, gds:$gds),
235  " $vdst, $addr, $data0, $data1$offset$gds"> {
236
237  let hasPostISelHook = 1;
238  let IsAtomicRet = 1;
239}
240
241multiclass DS_1A2D_RET_mc<string opName,
242                          RegisterClass rc = VGPR_32,
243                          string NoRetOp = "",
244                          RegisterClass src = rc> {
245  def "" : DS_1A2D_RET<opName, rc, src>,
246    AtomicNoRet<NoRetOp, !ne(NoRetOp, "")>;
247
248  let has_m0_read = 0 in {
249    def _gfx9 : DS_1A2D_RET<opName, rc, src>,
250      AtomicNoRet<NoRetOp#"_gfx9", !ne(NoRetOp, "")>;
251  }
252}
253
254class DS_1A2D_Off8_RET<string opName,
255                       RegisterClass rc = VGPR_32,
256                       RegisterClass src = rc,
257                       RegisterOperand dst_op = getLdStRegisterOperand<rc>.ret,
258                       RegisterOperand src_op = getLdStRegisterOperand<src>.ret>
259: DS_Pseudo<opName,
260  (outs dst_op:$vdst),
261  (ins VGPR_32:$addr, src_op:$data0, src_op:$data1, offset0:$offset0, offset1:$offset1, gds:$gds),
262  " $vdst, $addr, $data0, $data1$offset0$offset1$gds"> {
263
264  let has_offset = 0;
265  let AsmMatchConverter = "cvtDSOffset01";
266
267  let hasPostISelHook = 1;
268}
269
270multiclass DS_1A2D_Off8_RET_mc<string opName,
271                               RegisterClass rc = VGPR_32,
272                               RegisterClass src = rc> {
273  def "" : DS_1A2D_Off8_RET<opName, rc, src>;
274
275  let has_m0_read = 0 in {
276    def _gfx9 : DS_1A2D_Off8_RET<opName, rc, src>;
277  }
278}
279
280
281class DS_1A_RET<string opName, RegisterClass rc = VGPR_32, bit HasTiedOutput = 0, Operand ofs = offset,
282                RegisterOperand data_op = getLdStRegisterOperand<rc>.ret>
283: DS_Pseudo<opName,
284  (outs data_op:$vdst),
285  !if(HasTiedOutput,
286    (ins VGPR_32:$addr, ofs:$offset, gds:$gds, data_op:$vdst_in),
287    (ins VGPR_32:$addr, ofs:$offset, gds:$gds)),
288  " $vdst, $addr$offset$gds"> {
289  let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", "");
290  let DisableEncoding = !if(HasTiedOutput, "$vdst_in", "");
291  let has_data0 = 0;
292  let has_data1 = 0;
293}
294
295multiclass DS_1A_RET_mc<string opName, RegisterClass rc = VGPR_32, bit HasTiedOutput = 0, Operand ofs = offset> {
296  def "" : DS_1A_RET<opName, rc, HasTiedOutput, ofs>;
297
298  let has_m0_read = 0 in {
299    def _gfx9 : DS_1A_RET<opName, rc, HasTiedOutput, ofs>;
300  }
301}
302
303class DS_1A_RET_Tied<string opName, RegisterClass rc = VGPR_32> :
304  DS_1A_RET<opName, rc, 1>;
305
306class DS_1A_Off8_RET <string opName, RegisterClass rc = VGPR_32>
307: DS_Pseudo<opName,
308  (outs getLdStRegisterOperand<rc>.ret:$vdst),
309  (ins VGPR_32:$addr, offset0:$offset0, offset1:$offset1, gds:$gds),
310  " $vdst, $addr$offset0$offset1$gds"> {
311
312  let has_offset = 0;
313  let has_data0 = 0;
314  let has_data1 = 0;
315  let AsmMatchConverter = "cvtDSOffset01";
316}
317
318multiclass DS_1A_Off8_RET_mc <string opName, RegisterClass rc = VGPR_32> {
319  def "" : DS_1A_Off8_RET<opName, rc>;
320
321  let has_m0_read = 0 in {
322    def _gfx9 : DS_1A_Off8_RET<opName, rc>;
323  }
324}
325
326class DS_1A_RET_GDS <string opName> : DS_Pseudo<opName,
327  (outs getLdStRegisterOperand<VGPR_32>.ret:$vdst),
328  (ins VGPR_32:$addr, offset:$offset),
329  " $vdst, $addr$offset gds"> {
330
331  let has_data0 = 0;
332  let has_data1 = 0;
333  let has_gds = 0;
334  let gdsValue = 1;
335  let AsmMatchConverter = "cvtDSGds";
336}
337
338class DS_0A_RET <string opName> : DS_Pseudo<opName,
339  (outs getLdStRegisterOperand<VGPR_32>.ret:$vdst),
340  (ins offset:$offset, gds:$gds),
341  " $vdst$offset$gds"> {
342
343  let mayLoad = 1;
344  let mayStore = 1;
345
346  let has_addr = 0;
347  let has_data0 = 0;
348  let has_data1 = 0;
349}
350
351class DS_1A <string opName> : DS_Pseudo<opName,
352  (outs),
353  (ins VGPR_32:$addr, offset:$offset, gds:$gds),
354  " $addr$offset$gds"> {
355
356  let mayLoad = 1;
357  let mayStore = 1;
358
359  let has_vdst = 0;
360  let has_data0 = 0;
361  let has_data1 = 0;
362}
363
364multiclass DS_1A_mc <string opName> {
365  def "" : DS_1A<opName>;
366
367  let has_m0_read = 0 in {
368    def _gfx9 : DS_1A<opName>;
369  }
370}
371
372
373class DS_GWS <string opName, dag ins, string asmOps>
374: DS_Pseudo<opName, (outs), ins, asmOps> {
375
376  let has_vdst  = 0;
377  let has_addr  = 0;
378  let has_data0 = 0;
379  let has_data1 = 0;
380
381  let has_gds   = 0;
382  let gdsValue  = 1;
383  let AsmMatchConverter = "cvtDSGds";
384}
385
386class DS_GWS_0D <string opName>
387: DS_GWS<opName,
388  (ins offset:$offset), "$offset gds"> {
389  let hasSideEffects = 1;
390}
391
392class DS_GWS_1D <string opName>
393: DS_GWS<opName,
394  (ins getLdStRegisterOperand<VGPR_32>.ret:$data0, offset:$offset),
395  " $data0$offset gds"> {
396
397  let has_gws_data0 = 1;
398  let hasSideEffects = 1;
399}
400
401class DS_VOID <string opName> : DS_Pseudo<opName,
402  (outs), (ins), ""> {
403  let mayLoad = 0;
404  let mayStore = 0;
405  let hasSideEffects = 1;
406  let UseNamedOperandTable = 0;
407  let AsmMatchConverter = "";
408
409  let has_vdst = 0;
410  let has_addr = 0;
411  let has_data0 = 0;
412  let has_data1 = 0;
413  let has_offset = 0;
414  let has_offset0 = 0;
415  let has_offset1 = 0;
416  let has_gds = 0;
417}
418
419class DS_1A1D_PERMUTE <string opName, SDPatternOperator node = null_frag,
420                       RegisterOperand data_op = getLdStRegisterOperand<VGPR_32>.ret>
421: DS_Pseudo<opName,
422  (outs data_op:$vdst),
423  (ins VGPR_32:$addr, data_op:$data0, offset:$offset),
424  " $vdst, $addr, $data0$offset",
425  [(set i32:$vdst,
426   (node (DS1Addr1Offset i32:$addr, i16:$offset), i32:$data0))] > {
427
428  let mayLoad = 0;
429  let mayStore = 0;
430  let isConvergent = 1;
431
432  let has_data1 = 0;
433  let has_gds = 0;
434}
435
436defm DS_ADD_U32       : DS_1A1D_NORET_mc<"ds_add_u32">;
437defm DS_SUB_U32       : DS_1A1D_NORET_mc<"ds_sub_u32">;
438defm DS_RSUB_U32      : DS_1A1D_NORET_mc<"ds_rsub_u32">;
439defm DS_INC_U32       : DS_1A1D_NORET_mc<"ds_inc_u32">;
440defm DS_DEC_U32       : DS_1A1D_NORET_mc<"ds_dec_u32">;
441defm DS_MIN_I32       : DS_1A1D_NORET_mc<"ds_min_i32">;
442defm DS_MAX_I32       : DS_1A1D_NORET_mc<"ds_max_i32">;
443defm DS_MIN_U32       : DS_1A1D_NORET_mc<"ds_min_u32">;
444defm DS_MAX_U32       : DS_1A1D_NORET_mc<"ds_max_u32">;
445defm DS_AND_B32       : DS_1A1D_NORET_mc<"ds_and_b32">;
446defm DS_OR_B32        : DS_1A1D_NORET_mc<"ds_or_b32">;
447defm DS_XOR_B32       : DS_1A1D_NORET_mc<"ds_xor_b32">;
448
449let SubtargetPredicate = HasLDSFPAtomicAdd in {
450defm DS_ADD_F32       : DS_1A1D_NORET_mc<"ds_add_f32">;
451}
452
453defm DS_MIN_F32       : DS_1A1D_NORET_mc<"ds_min_f32">;
454defm DS_MAX_F32       : DS_1A1D_NORET_mc<"ds_max_f32">;
455
456let mayLoad = 0 in {
457defm DS_WRITE_B8      : DS_1A1D_NORET_mc<"ds_write_b8">;
458defm DS_WRITE_B16     : DS_1A1D_NORET_mc<"ds_write_b16">;
459defm DS_WRITE_B32     : DS_1A1D_NORET_mc<"ds_write_b32">;
460defm DS_WRITE2_B32    : DS_1A2D_Off8_NORET_mc<"ds_write2_b32">;
461defm DS_WRITE2ST64_B32: DS_1A2D_Off8_NORET_mc<"ds_write2st64_b32">;
462
463
464let has_m0_read = 0 in {
465
466let SubtargetPredicate = HasD16LoadStore in {
467def DS_WRITE_B8_D16_HI  : DS_1A1D_NORET<"ds_write_b8_d16_hi">;
468def DS_WRITE_B16_D16_HI : DS_1A1D_NORET<"ds_write_b16_d16_hi">;
469}
470
471} // End has_m0_read = 0
472
473let SubtargetPredicate = HasDSAddTid in {
474def DS_WRITE_ADDTID_B32 : DS_0A1D_NORET<"ds_write_addtid_b32">;
475}
476
477} // End mayLoad = 0
478
479let SubtargetPredicate = isGFX90APlus in {
480  defm DS_ADD_F64     : DS_1A1D_NORET_mc_gfx9<"ds_add_f64", VReg_64>;
481  defm DS_ADD_RTN_F64 : DS_1A1D_RET_mc_gfx9<"ds_add_rtn_f64", VReg_64, "ds_add_f64">;
482} // End SubtargetPredicate = isGFX90APlus
483
484let SubtargetPredicate = isGFX940Plus in {
485  defm DS_PK_ADD_F16      : DS_1A1D_NORET_mc_gfx9<"ds_pk_add_f16">;
486  defm DS_PK_ADD_RTN_F16  : DS_1A1D_RET_mc_gfx9<"ds_pk_add_rtn_f16", VGPR_32, "ds_pk_add_f16">;
487  defm DS_PK_ADD_BF16     : DS_1A1D_NORET_mc_gfx9<"ds_pk_add_bf16">;
488  defm DS_PK_ADD_RTN_BF16 : DS_1A1D_RET_mc_gfx9<"ds_pk_add_rtn_bf16", VGPR_32, "ds_pk_add_bf16">;
489} // End SubtargetPredicate = isGFX940Plus
490
491defm DS_CMPSTORE_B32     : DS_1A2D_NORET_mc<"ds_cmpstore_b32">;
492defm DS_CMPSTORE_F32     : DS_1A2D_NORET_mc<"ds_cmpstore_f32">;
493defm DS_CMPSTORE_B64     : DS_1A2D_NORET_mc<"ds_cmpstore_b64", VReg_64>;
494defm DS_CMPSTORE_F64     : DS_1A2D_NORET_mc<"ds_cmpstore_f64", VReg_64>;
495defm DS_CMPSTORE_RTN_B32 : DS_1A2D_RET_mc<"ds_cmpstore_rtn_b32", VGPR_32, "ds_cmpstore_b32">;
496defm DS_CMPSTORE_RTN_F32 : DS_1A2D_RET_mc<"ds_cmpstore_rtn_f32", VGPR_32, "ds_cmpstore_f32">;
497defm DS_CMPSTORE_RTN_B64  : DS_1A2D_RET_mc<"ds_cmpstore_rtn_b64", VReg_64, "ds_cmpstore_b64">;
498defm DS_CMPSTORE_RTN_F64  : DS_1A2D_RET_mc<"ds_cmpstore_rtn_f64", VReg_64, "ds_cmpstore_f64">;
499
500defm DS_MSKOR_B32     : DS_1A2D_NORET_mc<"ds_mskor_b32">;
501defm DS_CMPST_B32     : DS_1A2D_NORET_mc<"ds_cmpst_b32">;
502defm DS_CMPST_F32     : DS_1A2D_NORET_mc<"ds_cmpst_f32">;
503
504defm DS_ADD_U64       : DS_1A1D_NORET_mc<"ds_add_u64", VReg_64>;
505defm DS_SUB_U64       : DS_1A1D_NORET_mc<"ds_sub_u64", VReg_64>;
506defm DS_RSUB_U64      : DS_1A1D_NORET_mc<"ds_rsub_u64", VReg_64>;
507defm DS_INC_U64       : DS_1A1D_NORET_mc<"ds_inc_u64", VReg_64>;
508defm DS_DEC_U64       : DS_1A1D_NORET_mc<"ds_dec_u64", VReg_64>;
509defm DS_MIN_I64       : DS_1A1D_NORET_mc<"ds_min_i64", VReg_64>;
510defm DS_MAX_I64       : DS_1A1D_NORET_mc<"ds_max_i64", VReg_64>;
511defm DS_MIN_U64       : DS_1A1D_NORET_mc<"ds_min_u64", VReg_64>;
512defm DS_MAX_U64       : DS_1A1D_NORET_mc<"ds_max_u64", VReg_64>;
513defm DS_AND_B64       : DS_1A1D_NORET_mc<"ds_and_b64", VReg_64>;
514defm DS_OR_B64        : DS_1A1D_NORET_mc<"ds_or_b64", VReg_64>;
515defm DS_XOR_B64       : DS_1A1D_NORET_mc<"ds_xor_b64", VReg_64>;
516defm DS_MSKOR_B64     : DS_1A2D_NORET_mc<"ds_mskor_b64", VReg_64>;
517let mayLoad = 0 in {
518defm DS_WRITE_B64     : DS_1A1D_NORET_mc<"ds_write_b64", VReg_64>;
519defm DS_WRITE2_B64    : DS_1A2D_Off8_NORET_mc<"ds_write2_b64", VReg_64>;
520defm DS_WRITE2ST64_B64: DS_1A2D_Off8_NORET_mc<"ds_write2st64_b64", VReg_64>;
521}
522defm DS_CMPST_B64     : DS_1A2D_NORET_mc<"ds_cmpst_b64", VReg_64>;
523defm DS_CMPST_F64     : DS_1A2D_NORET_mc<"ds_cmpst_f64", VReg_64>;
524defm DS_MIN_F64       : DS_1A1D_NORET_mc<"ds_min_f64", VReg_64>;
525defm DS_MAX_F64       : DS_1A1D_NORET_mc<"ds_max_f64", VReg_64>;
526
527defm DS_ADD_RTN_U32   : DS_1A1D_RET_mc<"ds_add_rtn_u32", VGPR_32, "ds_add_u32">;
528
529let SubtargetPredicate = HasLDSFPAtomicAdd in {
530defm DS_ADD_RTN_F32   : DS_1A1D_RET_mc<"ds_add_rtn_f32", VGPR_32, "ds_add_f32">;
531}
532defm DS_SUB_RTN_U32   : DS_1A1D_RET_mc<"ds_sub_rtn_u32", VGPR_32, "ds_sub_u32">;
533defm DS_RSUB_RTN_U32  : DS_1A1D_RET_mc<"ds_rsub_rtn_u32", VGPR_32, "ds_rsub_u32">;
534defm DS_INC_RTN_U32   : DS_1A1D_RET_mc<"ds_inc_rtn_u32", VGPR_32, "ds_inc_u32">;
535defm DS_DEC_RTN_U32   : DS_1A1D_RET_mc<"ds_dec_rtn_u32", VGPR_32, "ds_dec_u32">;
536defm DS_MIN_RTN_I32   : DS_1A1D_RET_mc<"ds_min_rtn_i32", VGPR_32, "ds_min_i32">;
537defm DS_MAX_RTN_I32   : DS_1A1D_RET_mc<"ds_max_rtn_i32", VGPR_32, "ds_max_i32">;
538defm DS_MIN_RTN_U32   : DS_1A1D_RET_mc<"ds_min_rtn_u32", VGPR_32, "ds_min_u32">;
539defm DS_MAX_RTN_U32   : DS_1A1D_RET_mc<"ds_max_rtn_u32", VGPR_32, "ds_max_u32">;
540defm DS_AND_RTN_B32   : DS_1A1D_RET_mc<"ds_and_rtn_b32", VGPR_32, "ds_and_b32">;
541defm DS_OR_RTN_B32    : DS_1A1D_RET_mc<"ds_or_rtn_b32", VGPR_32, "ds_or_b32">;
542defm DS_XOR_RTN_B32   : DS_1A1D_RET_mc<"ds_xor_rtn_b32", VGPR_32, "ds_xor_b32">;
543defm DS_MSKOR_RTN_B32 : DS_1A2D_RET_mc<"ds_mskor_rtn_b32", VGPR_32, "ds_mskor_b32">;
544defm DS_CMPST_RTN_B32 : DS_1A2D_RET_mc<"ds_cmpst_rtn_b32", VGPR_32, "ds_cmpst_b32">;
545defm DS_CMPST_RTN_F32 : DS_1A2D_RET_mc<"ds_cmpst_rtn_f32", VGPR_32, "ds_cmpst_f32">;
546defm DS_MIN_RTN_F32   : DS_1A1D_RET_mc<"ds_min_rtn_f32", VGPR_32, "ds_min_f32">;
547defm DS_MAX_RTN_F32   : DS_1A1D_RET_mc<"ds_max_rtn_f32", VGPR_32, "ds_max_f32">;
548
549defm DS_WRXCHG_RTN_B32 : DS_1A1D_RET_mc<"ds_wrxchg_rtn_b32">;
550defm DS_WRXCHG2_RTN_B32 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2_rtn_b32", VReg_64, VGPR_32>;
551defm DS_WRXCHG2ST64_RTN_B32 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2st64_rtn_b32", VReg_64, VGPR_32>;
552
553defm DS_ADD_RTN_U64  : DS_1A1D_RET_mc<"ds_add_rtn_u64", VReg_64, "ds_add_u64">;
554defm DS_SUB_RTN_U64  : DS_1A1D_RET_mc<"ds_sub_rtn_u64", VReg_64, "ds_sub_u64">;
555defm DS_RSUB_RTN_U64  : DS_1A1D_RET_mc<"ds_rsub_rtn_u64", VReg_64, "ds_rsub_u64">;
556defm DS_INC_RTN_U64   : DS_1A1D_RET_mc<"ds_inc_rtn_u64", VReg_64, "ds_inc_u64">;
557defm DS_DEC_RTN_U64   : DS_1A1D_RET_mc<"ds_dec_rtn_u64", VReg_64, "ds_dec_u64">;
558defm DS_MIN_RTN_I64    : DS_1A1D_RET_mc<"ds_min_rtn_i64", VReg_64, "ds_min_i64">;
559defm DS_MAX_RTN_I64    : DS_1A1D_RET_mc<"ds_max_rtn_i64", VReg_64, "ds_max_i64">;
560defm DS_MIN_RTN_U64   : DS_1A1D_RET_mc<"ds_min_rtn_u64", VReg_64, "ds_min_u64">;
561defm DS_MAX_RTN_U64   : DS_1A1D_RET_mc<"ds_max_rtn_u64", VReg_64, "ds_max_u64">;
562defm DS_AND_RTN_B64    : DS_1A1D_RET_mc<"ds_and_rtn_b64", VReg_64, "ds_and_b64">;
563defm DS_OR_RTN_B64     : DS_1A1D_RET_mc<"ds_or_rtn_b64", VReg_64, "ds_or_b64">;
564defm DS_XOR_RTN_B64    : DS_1A1D_RET_mc<"ds_xor_rtn_b64", VReg_64, "ds_xor_b64">;
565defm DS_MSKOR_RTN_B64  : DS_1A2D_RET_mc<"ds_mskor_rtn_b64", VReg_64, "ds_mskor_b64">;
566defm DS_CMPST_RTN_B64  : DS_1A2D_RET_mc<"ds_cmpst_rtn_b64", VReg_64, "ds_cmpst_b64">;
567defm DS_CMPST_RTN_F64  : DS_1A2D_RET_mc<"ds_cmpst_rtn_f64", VReg_64, "ds_cmpst_f64">;
568defm DS_MIN_RTN_F64    : DS_1A1D_RET_mc<"ds_min_rtn_f64", VReg_64, "ds_min_f64">;
569defm DS_MAX_RTN_F64    : DS_1A1D_RET_mc<"ds_max_rtn_f64", VReg_64, "ds_max_f64">;
570
571defm DS_WRXCHG_RTN_B64 : DS_1A1D_RET_mc<"ds_wrxchg_rtn_b64", VReg_64>;
572defm DS_WRXCHG2_RTN_B64 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2_rtn_b64", VReg_128, VReg_64>;
573defm DS_WRXCHG2ST64_RTN_B64 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2st64_rtn_b64", VReg_128, VReg_64>;
574
575let isConvergent = 1, usesCustomInserter = 1 in {
576def DS_GWS_INIT       : DS_GWS_1D<"ds_gws_init"> {
577  let mayLoad = 0;
578}
579def DS_GWS_SEMA_V     : DS_GWS_0D<"ds_gws_sema_v">;
580def DS_GWS_SEMA_BR    : DS_GWS_1D<"ds_gws_sema_br">;
581def DS_GWS_SEMA_P     : DS_GWS_0D<"ds_gws_sema_p">;
582def DS_GWS_BARRIER    : DS_GWS_1D<"ds_gws_barrier">;
583}
584
585let SubtargetPredicate = HasDsSrc2Insts in {
586def DS_ADD_SRC2_U32   : DS_1A<"ds_add_src2_u32">;
587def DS_SUB_SRC2_U32   : DS_1A<"ds_sub_src2_u32">;
588def DS_RSUB_SRC2_U32  : DS_1A<"ds_rsub_src2_u32">;
589def DS_INC_SRC2_U32   : DS_1A<"ds_inc_src2_u32">;
590def DS_DEC_SRC2_U32   : DS_1A<"ds_dec_src2_u32">;
591def DS_MIN_SRC2_I32   : DS_1A<"ds_min_src2_i32">;
592def DS_MAX_SRC2_I32   : DS_1A<"ds_max_src2_i32">;
593def DS_MIN_SRC2_U32   : DS_1A<"ds_min_src2_u32">;
594def DS_MAX_SRC2_U32   : DS_1A<"ds_max_src2_u32">;
595def DS_AND_SRC2_B32   : DS_1A<"ds_and_src2_b32">;
596def DS_OR_SRC2_B32    : DS_1A<"ds_or_src2_b32">;
597def DS_XOR_SRC2_B32   : DS_1A<"ds_xor_src2_b32">;
598def DS_MIN_SRC2_F32   : DS_1A<"ds_min_src2_f32">;
599def DS_MAX_SRC2_F32   : DS_1A<"ds_max_src2_f32">;
600
601def DS_ADD_SRC2_U64   : DS_1A<"ds_add_src2_u64">;
602def DS_SUB_SRC2_U64   : DS_1A<"ds_sub_src2_u64">;
603def DS_RSUB_SRC2_U64  : DS_1A<"ds_rsub_src2_u64">;
604def DS_INC_SRC2_U64   : DS_1A<"ds_inc_src2_u64">;
605def DS_DEC_SRC2_U64   : DS_1A<"ds_dec_src2_u64">;
606def DS_MIN_SRC2_I64   : DS_1A<"ds_min_src2_i64">;
607def DS_MAX_SRC2_I64   : DS_1A<"ds_max_src2_i64">;
608def DS_MIN_SRC2_U64   : DS_1A<"ds_min_src2_u64">;
609def DS_MAX_SRC2_U64   : DS_1A<"ds_max_src2_u64">;
610def DS_AND_SRC2_B64   : DS_1A<"ds_and_src2_b64">;
611def DS_OR_SRC2_B64    : DS_1A<"ds_or_src2_b64">;
612def DS_XOR_SRC2_B64   : DS_1A<"ds_xor_src2_b64">;
613def DS_MIN_SRC2_F64   : DS_1A<"ds_min_src2_f64">;
614def DS_MAX_SRC2_F64   : DS_1A<"ds_max_src2_f64">;
615
616def DS_WRITE_SRC2_B32 : DS_1A<"ds_write_src2_b32">;
617def DS_WRITE_SRC2_B64 : DS_1A<"ds_write_src2_b64">;
618} // End SubtargetPredicate = HasDsSrc2Insts
619
620let Uses = [EXEC], mayLoad = 0, mayStore = 0, isConvergent = 1 in {
621def DS_SWIZZLE_B32 : DS_1A_RET <"ds_swizzle_b32", VGPR_32, 0, SwizzleImm>;
622}
623
624let mayStore = 0 in {
625defm DS_READ_I8      : DS_1A_RET_mc<"ds_read_i8">;
626defm DS_READ_U8      : DS_1A_RET_mc<"ds_read_u8">;
627defm DS_READ_I16     : DS_1A_RET_mc<"ds_read_i16">;
628defm DS_READ_U16     : DS_1A_RET_mc<"ds_read_u16">;
629defm DS_READ_B32     : DS_1A_RET_mc<"ds_read_b32">;
630defm DS_READ_B64     : DS_1A_RET_mc<"ds_read_b64", VReg_64>;
631
632defm DS_READ2_B32    : DS_1A_Off8_RET_mc<"ds_read2_b32", VReg_64>;
633defm DS_READ2ST64_B32: DS_1A_Off8_RET_mc<"ds_read2st64_b32", VReg_64>;
634
635defm DS_READ2_B64    : DS_1A_Off8_RET_mc<"ds_read2_b64", VReg_128>;
636defm DS_READ2ST64_B64: DS_1A_Off8_RET_mc<"ds_read2st64_b64", VReg_128>;
637
638let has_m0_read = 0 in {
639let SubtargetPredicate = HasD16LoadStore in {
640def DS_READ_U8_D16     : DS_1A_RET_Tied<"ds_read_u8_d16">;
641def DS_READ_U8_D16_HI  : DS_1A_RET_Tied<"ds_read_u8_d16_hi">;
642def DS_READ_I8_D16     : DS_1A_RET_Tied<"ds_read_i8_d16">;
643def DS_READ_I8_D16_HI  : DS_1A_RET_Tied<"ds_read_i8_d16_hi">;
644def DS_READ_U16_D16    : DS_1A_RET_Tied<"ds_read_u16_d16">;
645def DS_READ_U16_D16_HI : DS_1A_RET_Tied<"ds_read_u16_d16_hi">;
646}
647} // End has_m0_read = 0
648
649let SubtargetPredicate = HasDSAddTid in {
650def DS_READ_ADDTID_B32 : DS_0A_RET<"ds_read_addtid_b32">;
651}
652
653} // End mayStore = 0
654
655def DS_CONSUME       : DS_0A_RET<"ds_consume">;
656def DS_APPEND        : DS_0A_RET<"ds_append">;
657
658let SubtargetPredicate = isNotGFX90APlus in
659def DS_ORDERED_COUNT : DS_1A_RET_GDS<"ds_ordered_count">;
660
661//===----------------------------------------------------------------------===//
662// Instruction definitions for CI and newer.
663//===----------------------------------------------------------------------===//
664
665let SubtargetPredicate = isGFX7Plus in {
666
667defm DS_WRAP_RTN_B32 : DS_1A2D_RET_mc<"ds_wrap_rtn_b32", VGPR_32>;
668defm DS_CONDXCHG32_RTN_B64 : DS_1A1D_RET_mc<"ds_condxchg32_rtn_b64", VReg_64>;
669
670let isConvergent = 1, usesCustomInserter = 1 in {
671def DS_GWS_SEMA_RELEASE_ALL : DS_GWS_0D<"ds_gws_sema_release_all">;
672}
673
674let mayStore = 0 in {
675defm DS_READ_B96 : DS_1A_RET_mc<"ds_read_b96", VReg_96>;
676defm DS_READ_B128: DS_1A_RET_mc<"ds_read_b128", VReg_128>;
677} // End mayStore = 0
678
679let mayLoad = 0 in {
680defm DS_WRITE_B96 : DS_1A1D_NORET_mc<"ds_write_b96", VReg_96>;
681defm DS_WRITE_B128 : DS_1A1D_NORET_mc<"ds_write_b128", VReg_128>;
682} // End mayLoad = 0
683
684def DS_NOP : DS_VOID<"ds_nop">;
685
686} // let SubtargetPredicate = isGFX7Plus
687
688//===----------------------------------------------------------------------===//
689// Instruction definitions for VI and newer.
690//===----------------------------------------------------------------------===//
691
692let SubtargetPredicate = isGFX8Plus in {
693
694let Uses = [EXEC] in {
695def DS_PERMUTE_B32  : DS_1A1D_PERMUTE <"ds_permute_b32",
696                                       int_amdgcn_ds_permute>;
697def DS_BPERMUTE_B32 : DS_1A1D_PERMUTE <"ds_bpermute_b32",
698                                       int_amdgcn_ds_bpermute>;
699}
700
701} // let SubtargetPredicate = isGFX8Plus
702
703let SubtargetPredicate = HasLDSFPAtomicAdd, OtherPredicates = [HasDsSrc2Insts] in {
704def DS_ADD_SRC2_F32 : DS_1A<"ds_add_src2_f32">;
705}
706
707
708//===----------------------------------------------------------------------===//
709// Instruction definitions for GFX11 and newer.
710//===----------------------------------------------------------------------===//
711
712let SubtargetPredicate = isGFX11Plus in {
713
714def DS_ADD_GS_REG_RTN : DS_0A1D_RET_GDS<"ds_add_gs_reg_rtn", VReg_64, VGPR_32>;
715def DS_SUB_GS_REG_RTN : DS_0A1D_RET_GDS<"ds_sub_gs_reg_rtn", VReg_64, VGPR_32>;
716
717} // let SubtargetPredicate = isGFX11Plus
718
719//===----------------------------------------------------------------------===//
720// DS Patterns
721//===----------------------------------------------------------------------===//
722
723def : GCNPat <
724  (int_amdgcn_ds_swizzle i32:$src, timm:$offset16),
725  (DS_SWIZZLE_B32 VGPR_32:$src, (as_i16timm $offset16), (i1 0))
726>;
727
728class DSReadPat <DS_Pseudo inst, ValueType vt, PatFrag frag, int gds=0> : GCNPat <
729  (vt (frag (DS1Addr1Offset i32:$ptr, i16:$offset))),
730  (inst $ptr, offset:$offset, (i1 gds))
731>;
732
733multiclass DSReadPat_mc<DS_Pseudo inst, ValueType vt, string frag> {
734
735  let OtherPredicates = [LDSRequiresM0Init] in {
736    def : DSReadPat<inst, vt, !cast<PatFrag>(frag#"_m0")>;
737  }
738
739  let OtherPredicates = [NotLDSRequiresM0Init] in {
740    def : DSReadPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
741  }
742}
743
744class DSReadPat_D16 <DS_Pseudo inst, PatFrag frag, ValueType vt> : GCNPat <
745  (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$in),
746  (inst $ptr, offset:$offset, (i1 0), $in)
747>;
748
749defm : DSReadPat_mc <DS_READ_I8, i32, "sextloadi8_local">;
750defm : DSReadPat_mc <DS_READ_I8,  i16, "sextloadi8_local">;
751defm : DSReadPat_mc <DS_READ_U8,  i32, "extloadi8_local">;
752defm : DSReadPat_mc <DS_READ_U8,  i32, "zextloadi8_local">;
753defm : DSReadPat_mc <DS_READ_U8,  i16, "extloadi8_local">;
754defm : DSReadPat_mc <DS_READ_U8,  i16, "zextloadi8_local">;
755defm : DSReadPat_mc <DS_READ_I16, i32, "sextloadi16_local">;
756defm : DSReadPat_mc <DS_READ_I16, i32, "sextloadi16_local">;
757defm : DSReadPat_mc <DS_READ_U16, i32, "extloadi16_local">;
758defm : DSReadPat_mc <DS_READ_U16, i32, "zextloadi16_local">;
759defm : DSReadPat_mc <DS_READ_U16, i16, "load_local">;
760
761foreach vt = Reg32Types.types in {
762defm : DSReadPat_mc <DS_READ_B32, vt, "load_local">;
763}
764
765defm : DSReadPat_mc <DS_READ_U8, i16, "atomic_load_8_local">;
766defm : DSReadPat_mc <DS_READ_U8, i32, "atomic_load_8_local">;
767defm : DSReadPat_mc <DS_READ_U16, i16, "atomic_load_16_local">;
768defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_16_local">;
769defm : DSReadPat_mc <DS_READ_B32, i32, "atomic_load_32_local">;
770defm : DSReadPat_mc <DS_READ_B64, i64, "atomic_load_64_local">;
771
772let OtherPredicates = [D16PreservesUnusedBits] in {
773def : DSReadPat_D16<DS_READ_U16_D16_HI, load_d16_hi_local, v2i16>;
774def : DSReadPat_D16<DS_READ_U16_D16_HI, load_d16_hi_local, v2f16>;
775def : DSReadPat_D16<DS_READ_U8_D16_HI, az_extloadi8_d16_hi_local, v2i16>;
776def : DSReadPat_D16<DS_READ_U8_D16_HI, az_extloadi8_d16_hi_local, v2f16>;
777def : DSReadPat_D16<DS_READ_I8_D16_HI, sextloadi8_d16_hi_local, v2i16>;
778def : DSReadPat_D16<DS_READ_I8_D16_HI, sextloadi8_d16_hi_local, v2f16>;
779
780def : DSReadPat_D16<DS_READ_U16_D16, load_d16_lo_local, v2i16>;
781def : DSReadPat_D16<DS_READ_U16_D16, load_d16_lo_local, v2f16>;
782def : DSReadPat_D16<DS_READ_U8_D16, az_extloadi8_d16_lo_local, v2i16>;
783def : DSReadPat_D16<DS_READ_U8_D16, az_extloadi8_d16_lo_local, v2f16>;
784def : DSReadPat_D16<DS_READ_I8_D16, sextloadi8_d16_lo_local, v2i16>;
785def : DSReadPat_D16<DS_READ_I8_D16, sextloadi8_d16_lo_local, v2f16>;
786}
787
788class DSWritePat <DS_Pseudo inst, ValueType vt, PatFrag frag, int gds=0> : GCNPat <
789  (frag vt:$value, (DS1Addr1Offset i32:$ptr, i16:$offset)),
790  (inst $ptr, getVregSrcForVT<vt>.ret:$value, offset:$offset, (i1 gds))
791>;
792
793multiclass DSWritePat_mc <DS_Pseudo inst, ValueType vt, string frag> {
794  let OtherPredicates = [LDSRequiresM0Init] in {
795    def : DSWritePat<inst, vt, !cast<PatFrag>(frag#"_m0")>;
796  }
797
798  let OtherPredicates = [NotLDSRequiresM0Init] in {
799    def : DSWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
800  }
801}
802
803// Irritatingly, atomic_store reverses the order of operands from a
804// normal store.
805class DSAtomicWritePat <DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat <
806  (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$value),
807  (inst $ptr, getVregSrcForVT<vt>.ret:$value, offset:$offset, (i1 0))
808>;
809
810multiclass DSAtomicWritePat_mc <DS_Pseudo inst, ValueType vt, string frag> {
811  let OtherPredicates = [LDSRequiresM0Init] in {
812    def : DSAtomicWritePat<inst, vt, !cast<PatFrag>(frag#"_m0")>;
813  }
814
815  let OtherPredicates = [NotLDSRequiresM0Init] in {
816    def : DSAtomicWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
817  }
818}
819
820defm : DSWritePat_mc <DS_WRITE_B8, i32, "truncstorei8_local">;
821defm : DSWritePat_mc <DS_WRITE_B16, i32, "truncstorei16_local">;
822defm : DSWritePat_mc <DS_WRITE_B8, i16, "truncstorei8_local">;
823defm : DSWritePat_mc <DS_WRITE_B16, i16, "store_local">;
824
825foreach vt = Reg32Types.types in {
826defm : DSWritePat_mc <DS_WRITE_B32, vt, "store_local">;
827}
828
829defm : DSAtomicWritePat_mc <DS_WRITE_B8, i16, "atomic_store_8_local">;
830defm : DSAtomicWritePat_mc <DS_WRITE_B8, i32, "atomic_store_8_local">;
831defm : DSAtomicWritePat_mc <DS_WRITE_B16, i16, "atomic_store_16_local">;
832defm : DSAtomicWritePat_mc <DS_WRITE_B16, i32, "atomic_store_16_local">;
833defm : DSAtomicWritePat_mc <DS_WRITE_B32, i32, "atomic_store_32_local">;
834defm : DSAtomicWritePat_mc <DS_WRITE_B64, i64, "atomic_store_64_local">;
835
836let OtherPredicates = [HasD16LoadStore] in {
837def : DSWritePat <DS_WRITE_B16_D16_HI, i32, store_hi16_local>;
838def : DSWritePat <DS_WRITE_B8_D16_HI, i32, truncstorei8_hi16_local>;
839}
840
841class DS64Bit4ByteAlignedReadPat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat <
842  (vt:$value (frag (DS64Bit4ByteAligned i32:$ptr, i8:$offset0, i8:$offset1))),
843  (inst $ptr, $offset0, $offset1, (i1 0))
844>;
845
846class DS64Bit4ByteAlignedWritePat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat<
847  (frag vt:$value, (DS64Bit4ByteAligned i32:$ptr, i8:$offset0, i8:$offset1)),
848  (inst $ptr, (i32 (EXTRACT_SUBREG VReg_64:$value, sub0)),
849              (i32 (EXTRACT_SUBREG VReg_64:$value, sub1)), $offset0, $offset1,
850              (i1 0))
851>;
852
853class DS128Bit8ByteAlignedReadPat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat <
854  (vt:$value (frag (DS128Bit8ByteAligned i32:$ptr, i8:$offset0, i8:$offset1))),
855  (inst $ptr, $offset0, $offset1, (i1 0))
856>;
857
858class DS128Bit8ByteAlignedWritePat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat<
859  (frag vt:$value, (DS128Bit8ByteAligned i32:$ptr, i8:$offset0, i8:$offset1)),
860  (inst $ptr, (i64 (EXTRACT_SUBREG VReg_128:$value, sub0_sub1)),
861              (i64 (EXTRACT_SUBREG VReg_128:$value, sub2_sub3)), $offset0, $offset1,
862              (i1 0))
863>;
864
865multiclass DS64Bit4ByteAlignedPat_mc<ValueType vt> {
866  let OtherPredicates = [LDSRequiresM0Init, isGFX7Plus] in {
867    def : DS64Bit4ByteAlignedReadPat<DS_READ2_B32, vt, load_local_m0>;
868    def : DS64Bit4ByteAlignedWritePat<DS_WRITE2_B32, vt, store_local_m0>;
869  }
870
871  let OtherPredicates = [NotLDSRequiresM0Init] in {
872    def : DS64Bit4ByteAlignedReadPat<DS_READ2_B32_gfx9, vt, load_local>;
873    def : DS64Bit4ByteAlignedWritePat<DS_WRITE2_B32_gfx9, vt, store_local>;
874  }
875}
876
877multiclass DS128Bit8ByteAlignedPat_mc<ValueType vt> {
878  let OtherPredicates = [LDSRequiresM0Init, isGFX7Plus] in {
879    def : DS128Bit8ByteAlignedReadPat<DS_READ2_B64, vt, load_local_m0>;
880    def : DS128Bit8ByteAlignedWritePat<DS_WRITE2_B64, vt, store_local_m0>;
881  }
882
883  let OtherPredicates = [NotLDSRequiresM0Init] in {
884    def : DS128Bit8ByteAlignedReadPat<DS_READ2_B64_gfx9, vt, load_local>;
885    def : DS128Bit8ByteAlignedWritePat<DS_WRITE2_B64_gfx9, vt, store_local>;
886  }
887}
888
889// v2i32 loads are split into i32 loads on SI during lowering, due to a bug
890// related to bounds checking.
891foreach vt = VReg_64.RegTypes in {
892defm : DS64Bit4ByteAlignedPat_mc<vt>;
893}
894
895foreach vt = VReg_128.RegTypes in {
896defm : DS128Bit8ByteAlignedPat_mc<vt>;
897}
898
899// Prefer ds_read over ds_read2 and ds_write over ds_write2, all other things
900// being equal, because it has a larger immediate offset range.
901let AddedComplexity = 100 in {
902
903foreach vt = VReg_64.RegTypes in {
904defm : DSReadPat_mc <DS_READ_B64, vt, "load_align8_local">;
905defm : DSWritePat_mc <DS_WRITE_B64, vt, "store_align8_local">;
906}
907
908let SubtargetPredicate = isGFX7Plus in {
909
910foreach vt = VReg_96.RegTypes in {
911defm : DSReadPat_mc <DS_READ_B96, vt, "load_align16_local">;
912defm : DSWritePat_mc <DS_WRITE_B96, vt, "store_align16_local">;
913}
914
915foreach vt = VReg_128.RegTypes in {
916defm : DSReadPat_mc <DS_READ_B128, vt, "load_align16_local">;
917defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_align16_local">;
918}
919
920let SubtargetPredicate = HasUnalignedAccessMode in {
921
922// Select 64 bit loads and stores aligned less than 4 as a single ds_read_b64/
923// ds_write_b64 instruction as this is faster than ds_read2_b32/ds_write2_b32
924// which would be used otherwise. In this case a b32 access would still be
925// misaligned, but we will have 2 of them.
926foreach vt = VReg_64.RegTypes in {
927defm : DSReadPat_mc <DS_READ_B64, vt, "load_align_less_than_4_local">;
928defm : DSWritePat_mc <DS_WRITE_B64, vt, "store_align_less_than_4_local">;
929}
930
931// Selection will split most of the unaligned 3 dword accesses due to performance
932// reasons when beneficial. Keep these two patterns for the rest of the cases.
933foreach vt = VReg_96.RegTypes in {
934defm : DSReadPat_mc <DS_READ_B96, vt, "load_local">;
935defm : DSWritePat_mc <DS_WRITE_B96, vt, "store_local">;
936}
937
938// Select 128 bit loads and stores aligned less than 4 as a single ds_read_b128/
939// ds_write_b128 instruction as this is faster than ds_read2_b64/ds_write2_b64
940// which would be used otherwise. In this case a b64 access would still be
941// misaligned, but we will have 2 of them.
942foreach vt = VReg_128.RegTypes in {
943defm : DSReadPat_mc <DS_READ_B128, vt, "load_align_less_than_4_local">;
944defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_align_less_than_4_local">;
945}
946
947} // End SubtargetPredicate = HasUnalignedAccessMode
948
949} // End SubtargetPredicate = isGFX7Plus
950
951} // End AddedComplexity = 100
952
953class DSAtomicRetPat<DS_Pseudo inst, ValueType vt, PatFrag frag, bit gds=0> : GCNPat <
954  (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$value),
955  (inst $ptr, getVregSrcForVT<vt>.ret:$value, offset:$offset, (i1 gds))
956>;
957
958multiclass DSAtomicRetPat_mc<DS_Pseudo inst, ValueType vt, string frag> {
959  let OtherPredicates = [LDSRequiresM0Init] in {
960    def : DSAtomicRetPat<inst, vt, !cast<PatFrag>(frag#"_local_m0_"#vt.Size)>;
961  }
962
963  let OtherPredicates = [NotLDSRequiresM0Init] in {
964    def : DSAtomicRetPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt,
965                         !cast<PatFrag>(frag#"_local_"#vt.Size)>;
966  }
967
968  def : DSAtomicRetPat<inst, vt, !cast<PatFrag>(frag#"_region_m0_"#vt.Size), 1>;
969}
970
971multiclass DSAtomicRetNoRetPat_mc<DS_Pseudo inst, DS_Pseudo noRetInst,
972                                  ValueType vt, string frag> {
973  let OtherPredicates = [LDSRequiresM0Init] in {
974    def : DSAtomicRetPat<inst, vt,
975                         !cast<PatFrag>(frag#"_local_m0_ret_"#vt.Size)>;
976    def : DSAtomicRetPat<noRetInst, vt,
977                         !cast<PatFrag>(frag#"_local_m0_noret_"#vt.Size)>;
978  }
979
980  let OtherPredicates = [NotLDSRequiresM0Init] in {
981    def : DSAtomicRetPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt,
982                         !cast<PatFrag>(frag#"_local_ret_"#vt.Size)>;
983    def : DSAtomicRetPat<!cast<DS_Pseudo>(!cast<string>(noRetInst)#"_gfx9"), vt,
984                         !cast<PatFrag>(frag#"_local_noret_"#vt.Size)>;
985  }
986
987  def : DSAtomicRetPat<inst, vt,
988                       !cast<PatFrag>(frag#"_region_m0_ret_"#vt.Size), 1>;
989  def : DSAtomicRetPat<noRetInst, vt,
990                       !cast<PatFrag>(frag#"_region_m0_noret_"#vt.Size), 1>;
991}
992
993
994
995let SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10 in {
996// Caution, the order of src and cmp is the *opposite* of the BUFFER_ATOMIC_CMPSWAP opcode.
997class DSAtomicCmpXChgSwapped<DS_Pseudo inst, ValueType vt, PatFrag frag, bit gds=0> : GCNPat <
998  (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$cmp, vt:$swap),
999  (inst $ptr, getVregSrcForVT<vt>.ret:$cmp, getVregSrcForVT<vt>.ret:$swap, offset:$offset, (i1 gds))
1000>;
1001
1002multiclass DSAtomicCmpXChgSwapped_mc<DS_Pseudo inst, DS_Pseudo noRetInst, ValueType vt,
1003                                     string frag> {
1004  let OtherPredicates = [LDSRequiresM0Init] in {
1005    def : DSAtomicCmpXChgSwapped<inst, vt, !cast<PatFrag>(frag#"_local_m0_ret_"#vt.Size)>;
1006    def : DSAtomicCmpXChgSwapped<noRetInst, vt, !cast<PatFrag>(frag#"_local_m0_noret_"#vt.Size)>;
1007  }
1008
1009  let OtherPredicates = [NotLDSRequiresM0Init] in {
1010    def : DSAtomicCmpXChgSwapped<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt,
1011                                 !cast<PatFrag>(frag#"_local_ret_"#vt.Size)>;
1012    def : DSAtomicCmpXChgSwapped<!cast<DS_Pseudo>(!cast<string>(noRetInst)#"_gfx9"), vt,
1013                                 !cast<PatFrag>(frag#"_local_noret_"#vt.Size)>;
1014  }
1015
1016  def : DSAtomicCmpXChgSwapped<inst, vt, !cast<PatFrag>(frag#"_region_m0_ret_"#vt.Size), 1>;
1017  def : DSAtomicCmpXChgSwapped<noRetInst, vt, !cast<PatFrag>(frag#"_region_m0_noret_"#vt.Size), 1>;
1018}
1019} // End SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10
1020
1021let SubtargetPredicate = isGFX11Plus in {
1022// The order of src and cmp agrees with the BUFFER_ATOMIC_CMPSWAP opcode.
1023class DSAtomicCmpXChg<DS_Pseudo inst, ValueType vt, PatFrag frag, bit gds=0> : GCNPat <
1024  (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$cmp, vt:$swap),
1025  (inst $ptr, getVregSrcForVT<vt>.ret:$swap, getVregSrcForVT<vt>.ret:$cmp, offset:$offset, (i1 gds))
1026>;
1027
1028multiclass DSAtomicCmpXChg_mc<DS_Pseudo inst, DS_Pseudo noRetInst, ValueType vt, string frag> {
1029
1030  def : DSAtomicCmpXChg<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt,
1031                        !cast<PatFrag>(frag#"_local_ret_"#vt.Size)>;
1032  def : DSAtomicCmpXChg<!cast<DS_Pseudo>(!cast<string>(noRetInst)#"_gfx9"), vt,
1033                        !cast<PatFrag>(frag#"_local_noret_"#vt.Size)>;
1034
1035  def : DSAtomicCmpXChg<inst, vt, !cast<PatFrag>(frag#"_region_m0_ret_"#vt.Size), 1>;
1036  def : DSAtomicCmpXChg<noRetInst, vt, !cast<PatFrag>(frag#"_region_m0_noret_"#vt.Size), 1>;
1037}
1038} // End SubtargetPredicate = isGFX11Plus
1039
1040// 32-bit atomics.
1041defm : DSAtomicRetPat_mc<DS_WRXCHG_RTN_B32, i32, "atomic_swap">;
1042defm : DSAtomicRetNoRetPat_mc<DS_ADD_RTN_U32, DS_ADD_U32, i32, "atomic_load_add">;
1043defm : DSAtomicRetNoRetPat_mc<DS_SUB_RTN_U32, DS_SUB_U32, i32, "atomic_load_sub">;
1044defm : DSAtomicRetNoRetPat_mc<DS_INC_RTN_U32, DS_INC_U32, i32, "atomic_inc">;
1045defm : DSAtomicRetNoRetPat_mc<DS_DEC_RTN_U32, DS_DEC_U32, i32, "atomic_dec">;
1046defm : DSAtomicRetNoRetPat_mc<DS_AND_RTN_B32, DS_AND_B32, i32, "atomic_load_and">;
1047defm : DSAtomicRetNoRetPat_mc<DS_OR_RTN_B32, DS_OR_B32, i32, "atomic_load_or">;
1048defm : DSAtomicRetNoRetPat_mc<DS_XOR_RTN_B32, DS_XOR_B32, i32, "atomic_load_xor">;
1049defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_I32, DS_MIN_I32, i32, "atomic_load_min">;
1050defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_I32, DS_MAX_I32, i32, "atomic_load_max">;
1051defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_U32, DS_MIN_U32, i32, "atomic_load_umin">;
1052defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_U32, DS_MAX_U32, i32, "atomic_load_umax">;
1053defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_F32, DS_MIN_F32, f32, "atomic_load_fmin">;
1054defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_F32, DS_MAX_F32, f32, "atomic_load_fmax">;
1055
1056let SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10 in {
1057defm : DSAtomicCmpXChgSwapped_mc<DS_CMPST_RTN_B32, DS_CMPST_B32, i32, "atomic_cmp_swap">;
1058}
1059
1060let SubtargetPredicate = isGFX11Plus in {
1061defm : DSAtomicCmpXChg_mc<DS_CMPSTORE_RTN_B32, DS_CMPSTORE_B32, i32, "atomic_cmp_swap">;
1062}
1063
1064let SubtargetPredicate = HasLDSFPAtomicAdd in {
1065defm : DSAtomicRetNoRetPat_mc<DS_ADD_RTN_F32, DS_ADD_F32, f32, "atomic_load_fadd">;
1066}
1067
1068// 64-bit atomics.
1069defm : DSAtomicRetPat_mc<DS_WRXCHG_RTN_B64, i64, "atomic_swap">;
1070defm : DSAtomicRetNoRetPat_mc<DS_ADD_RTN_U64, DS_ADD_U64, i64, "atomic_load_add">;
1071defm : DSAtomicRetNoRetPat_mc<DS_SUB_RTN_U64, DS_SUB_U64, i64, "atomic_load_sub">;
1072defm : DSAtomicRetNoRetPat_mc<DS_INC_RTN_U64, DS_INC_U64, i64, "atomic_inc">;
1073defm : DSAtomicRetNoRetPat_mc<DS_DEC_RTN_U64, DS_DEC_U64, i64, "atomic_dec">;
1074defm : DSAtomicRetNoRetPat_mc<DS_AND_RTN_B64, DS_AND_B64, i64, "atomic_load_and">;
1075defm : DSAtomicRetNoRetPat_mc<DS_OR_RTN_B64, DS_OR_B64, i64, "atomic_load_or">;
1076defm : DSAtomicRetNoRetPat_mc<DS_XOR_RTN_B64, DS_XOR_B64, i64, "atomic_load_xor">;
1077defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_I64, DS_MIN_I64, i64, "atomic_load_min">;
1078defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_I64, DS_MAX_I64, i64, "atomic_load_max">;
1079defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_U64, DS_MIN_U64, i64, "atomic_load_umin">;
1080defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_U64, DS_MAX_U64, i64, "atomic_load_umax">;
1081defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_F64, DS_MIN_F64, f64, "atomic_load_fmin">;
1082defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_F64, DS_MAX_F64, f64, "atomic_load_fmax">;
1083
1084let SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10 in {
1085defm : DSAtomicCmpXChgSwapped_mc<DS_CMPST_RTN_B64, DS_CMPST_B64, i64, "atomic_cmp_swap">;
1086} // End SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10
1087
1088let SubtargetPredicate = isGFX11Plus in {
1089defm : DSAtomicCmpXChg_mc<DS_CMPSTORE_RTN_B64, DS_CMPSTORE_B64, i64, "atomic_cmp_swap">;
1090} // End SubtargetPredicate = isGFX11Plus
1091
1092let SubtargetPredicate = isGFX90APlus in {
1093def : DSAtomicRetPat<DS_ADD_RTN_F64, f64, atomic_load_fadd_local_ret_64>;
1094def : DSAtomicRetPat<DS_ADD_F64, f64, atomic_load_fadd_local_noret_64>;
1095}
1096
1097let SubtargetPredicate = isGFX940Plus in {
1098def : DSAtomicRetPat<DS_PK_ADD_RTN_F16, v2f16, atomic_load_fadd_v2f16_local_ret_32>;
1099def : DSAtomicRetPat<DS_PK_ADD_F16, v2f16, atomic_load_fadd_v2f16_local_noret_32>;
1100def : GCNPat <
1101  (v2i16 (int_amdgcn_ds_fadd_v2bf16_ret i32:$ptr, v2i16:$src)),
1102  (DS_PK_ADD_RTN_BF16 VGPR_32:$ptr, VGPR_32:$src, 0, 0)
1103>;
1104def : GCNPat <
1105  (v2i16 (int_amdgcn_ds_fadd_v2bf16_noret i32:$ptr, v2i16:$src)),
1106  (DS_PK_ADD_BF16 VGPR_32:$ptr, VGPR_32:$src, 0, 0)
1107>;
1108}
1109
1110def : Pat <
1111  (SIds_ordered_count i32:$value, i16:$offset),
1112  (DS_ORDERED_COUNT $value, (as_i16imm $offset))
1113>;
1114
1115//===----------------------------------------------------------------------===//
1116// Target-specific instruction encodings.
1117//===----------------------------------------------------------------------===//
1118
1119//===----------------------------------------------------------------------===//
1120// Base ENC_DS for GFX6, GFX7, GFX10, GFX11.
1121//===----------------------------------------------------------------------===//
1122
1123class Base_DS_Real_gfx6_gfx7_gfx10_gfx11<bits<8> op, DS_Pseudo ps, int ef, string opName = ps.Mnemonic> :
1124    DS_Real<ps, opName>, SIMCInstr <ps.Mnemonic, ef> {
1125
1126  let Inst{7-0}   = !if(ps.has_offset0, offset0, 0);
1127  let Inst{15-8}  = !if(ps.has_offset1, offset1, 0);
1128  let Inst{17}    = !if(ps.has_gds, gds, ps.gdsValue);
1129  let Inst{25-18} = op;
1130  let Inst{31-26} = 0x36;
1131  let Inst{39-32} = !if(ps.has_addr, addr, !if(ps.has_gws_data0, data0{7-0}, 0));
1132  let Inst{47-40} = !if(ps.has_data0, data0{7-0}, 0);
1133  let Inst{55-48} = !if(ps.has_data1, data1{7-0}, 0);
1134  let Inst{63-56} = !if(ps.has_vdst, vdst{7-0}, 0);
1135}
1136
1137//===----------------------------------------------------------------------===//
1138// GFX11.
1139//===----------------------------------------------------------------------===//
1140
1141let AssemblerPredicate = isGFX11Plus, DecoderNamespace = "GFX11" in {
1142  multiclass DS_Real_gfx11<bits<8> op>  {
1143    def _gfx11 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, !cast<DS_Pseudo>(NAME),
1144                                              SIEncodingFamily.GFX11>;
1145  }
1146
1147  multiclass DS_Real_Renamed_gfx11<bits<8> op, DS_Pseudo backing_pseudo, string real_name> {
1148     def _gfx11 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, backing_pseudo, SIEncodingFamily.GFX11, real_name>,
1149               MnemonicAlias<backing_pseudo.Mnemonic, real_name>, Requires<[isGFX11Plus]>;
1150  }
1151} // End AssemblerPredicate = isGFX11Plus, DecoderNamespace = "GFX11"
1152
1153defm DS_STORE_B32                        : DS_Real_Renamed_gfx11<0x00d, DS_WRITE_B32, "ds_store_b32">;
1154defm DS_STORE_2ADDR_B32                  : DS_Real_Renamed_gfx11<0x00e, DS_WRITE2_B32, "ds_store_2addr_b32">;
1155defm DS_STORE_2ADDR_STRIDE64_B32         : DS_Real_Renamed_gfx11<0x00f, DS_WRITE2ST64_B32, "ds_store_2addr_stride64_b32">;
1156defm DS_STORE_B8                         : DS_Real_Renamed_gfx11<0x01e, DS_WRITE_B8, "ds_store_b8">;
1157defm DS_STORE_B16                        : DS_Real_Renamed_gfx11<0x01f, DS_WRITE_B16, "ds_store_b16">;
1158defm DS_STOREXCHG_RTN_B32                : DS_Real_Renamed_gfx11<0x02d, DS_WRXCHG_RTN_B32, "ds_storexchg_rtn_b32">;
1159defm DS_STOREXCHG_2ADDR_RTN_B32          : DS_Real_Renamed_gfx11<0x02e, DS_WRXCHG2_RTN_B32, "ds_storexchg_2addr_rtn_b32">;
1160defm DS_STOREXCHG_2ADDR_STRIDE64_RTN_B32 : DS_Real_Renamed_gfx11<0x02f, DS_WRXCHG2ST64_RTN_B32, "ds_storexchg_2addr_stride64_rtn_b32">;
1161defm DS_LOAD_B32                         : DS_Real_Renamed_gfx11<0x036, DS_READ_B32, "ds_load_b32">;
1162defm DS_LOAD_2ADDR_B32                   : DS_Real_Renamed_gfx11<0x037, DS_READ2_B32, "ds_load_2addr_b32">;
1163defm DS_LOAD_2ADDR_STRIDE64_B32          : DS_Real_Renamed_gfx11<0x038, DS_READ2ST64_B32, "ds_load_2addr_stride64_b32">;
1164defm DS_LOAD_I8                          : DS_Real_Renamed_gfx11<0x039, DS_READ_I8, "ds_load_i8">;
1165defm DS_LOAD_U8                          : DS_Real_Renamed_gfx11<0x03a, DS_READ_U8, "ds_load_u8">;
1166defm DS_LOAD_I16                         : DS_Real_Renamed_gfx11<0x03b, DS_READ_I16, "ds_load_i16">;
1167defm DS_LOAD_U16                         : DS_Real_Renamed_gfx11<0x03c, DS_READ_U16, "ds_load_u16">;
1168defm DS_STORE_B64                        : DS_Real_Renamed_gfx11<0x04d, DS_WRITE_B64, "ds_store_b64">;
1169defm DS_STORE_2ADDR_B64                  : DS_Real_Renamed_gfx11<0x04e, DS_WRITE2_B64, "ds_store_2addr_b64">;
1170defm DS_STORE_2ADDR_STRIDE64_B64         : DS_Real_Renamed_gfx11<0x04f, DS_WRITE2ST64_B64, "ds_store_2addr_stride64_b64">;
1171defm DS_STOREXCHG_RTN_B64                : DS_Real_Renamed_gfx11<0x06d, DS_WRXCHG_RTN_B64, "ds_storexchg_rtn_b64">;
1172defm DS_STOREXCHG_2ADDR_RTN_B64          : DS_Real_Renamed_gfx11<0x06e, DS_WRXCHG2_RTN_B64, "ds_storexchg_2addr_rtn_b64">;
1173defm DS_STOREXCHG_2ADDR_STRIDE64_RTN_B64 : DS_Real_Renamed_gfx11<0x06f, DS_WRXCHG2ST64_RTN_B64, "ds_storexchg_2addr_stride64_rtn_b64">;
1174defm DS_LOAD_B64                         : DS_Real_Renamed_gfx11<0x076, DS_READ_B64, "ds_load_b64">;
1175defm DS_LOAD_2ADDR_B64                   : DS_Real_Renamed_gfx11<0x077, DS_READ2_B64, "ds_load_2addr_b64">;
1176defm DS_LOAD_2ADDR_STRIDE64_B64          : DS_Real_Renamed_gfx11<0x078, DS_READ2ST64_B64, "ds_load_2addr_stride64_b64">;
1177defm DS_STORE_B8_D16_HI                  : DS_Real_Renamed_gfx11<0x0a0, DS_WRITE_B8_D16_HI, "ds_store_b8_d16_hi">;
1178defm DS_STORE_B16_D16_HI                 : DS_Real_Renamed_gfx11<0x0a1, DS_WRITE_B16_D16_HI, "ds_store_b16_d16_hi">;
1179defm DS_LOAD_U8_D16                      : DS_Real_Renamed_gfx11<0x0a2, DS_READ_U8_D16, "ds_load_u8_d16">;
1180defm DS_LOAD_U8_D16_HI                   : DS_Real_Renamed_gfx11<0x0a3, DS_READ_U8_D16_HI, "ds_load_u8_d16_hi">;
1181defm DS_LOAD_I8_D16                      : DS_Real_Renamed_gfx11<0x0a4, DS_READ_I8_D16, "ds_load_i8_d16">;
1182defm DS_LOAD_I8_D16_HI                   : DS_Real_Renamed_gfx11<0x0a5, DS_READ_I8_D16_HI, "ds_load_i8_d16_hi">;
1183defm DS_LOAD_U16_D16                     : DS_Real_Renamed_gfx11<0x0a6, DS_READ_U16_D16, "ds_load_u16_d16">;
1184defm DS_LOAD_U16_D16_HI                  : DS_Real_Renamed_gfx11<0x0a7, DS_READ_U16_D16_HI, "ds_load_u16_d16_hi">;
1185defm DS_STORE_ADDTID_B32                 : DS_Real_Renamed_gfx11<0x0b0, DS_WRITE_ADDTID_B32, "ds_store_addtid_b32">;
1186defm DS_LOAD_ADDTID_B32                  : DS_Real_Renamed_gfx11<0x0b1, DS_READ_ADDTID_B32, "ds_load_addtid_b32">;
1187defm DS_STORE_B96                        : DS_Real_Renamed_gfx11<0x0de, DS_WRITE_B96, "ds_store_b96">;
1188defm DS_STORE_B128                       : DS_Real_Renamed_gfx11<0x0df, DS_WRITE_B128, "ds_store_b128">;
1189defm DS_LOAD_B96                         : DS_Real_Renamed_gfx11<0x0fe, DS_READ_B96, "ds_load_b96">;
1190defm DS_LOAD_B128                        : DS_Real_Renamed_gfx11<0x0ff, DS_READ_B128, "ds_load_b128">;
1191
1192// DS_CMPST_* are renamed to DS_CMPSTORE_* in GFX11, but also the data operands (src and cmp) are swapped
1193// comparing to pre-GFX11.
1194// Note: the mnemonic alias is not generated to avoid a potential ambiguity due to the semantics change.
1195
1196defm DS_CMPSTORE_B32                     : DS_Real_gfx11<0x010>;
1197defm DS_CMPSTORE_F32                     : DS_Real_gfx11<0x011>;
1198defm DS_CMPSTORE_RTN_B32                 : DS_Real_gfx11<0x030>;
1199defm DS_CMPSTORE_RTN_F32                 : DS_Real_gfx11<0x031>;
1200defm DS_CMPSTORE_B64                     : DS_Real_gfx11<0x050>;
1201defm DS_CMPSTORE_F64                     : DS_Real_gfx11<0x051>;
1202defm DS_CMPSTORE_RTN_B64                 : DS_Real_gfx11<0x070>;
1203defm DS_CMPSTORE_RTN_F64                 : DS_Real_gfx11<0x071>;
1204
1205defm DS_ADD_RTN_F32                      : DS_Real_gfx11<0x079>;
1206defm DS_ADD_GS_REG_RTN                   : DS_Real_gfx11<0x07a>;
1207defm DS_SUB_GS_REG_RTN                   : DS_Real_gfx11<0x07b>;
1208
1209//===----------------------------------------------------------------------===//
1210// GFX10.
1211//===----------------------------------------------------------------------===//
1212
1213let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in {
1214  multiclass DS_Real_gfx10<bits<8> op>  {
1215    def _gfx10 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, !cast<DS_Pseudo>(NAME),
1216                                              SIEncodingFamily.GFX10>;
1217  }
1218} // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10"
1219
1220defm DS_ADD_RTN_F32      : DS_Real_gfx10<0x055>;
1221defm DS_WRITE_B8_D16_HI  : DS_Real_gfx10<0x0a0>;
1222defm DS_WRITE_B16_D16_HI : DS_Real_gfx10<0x0a1>;
1223defm DS_READ_U8_D16      : DS_Real_gfx10<0x0a2>;
1224defm DS_READ_U8_D16_HI   : DS_Real_gfx10<0x0a3>;
1225defm DS_READ_I8_D16      : DS_Real_gfx10<0x0a4>;
1226defm DS_READ_I8_D16_HI   : DS_Real_gfx10<0x0a5>;
1227defm DS_READ_U16_D16     : DS_Real_gfx10<0x0a6>;
1228defm DS_READ_U16_D16_HI  : DS_Real_gfx10<0x0a7>;
1229defm DS_WRITE_ADDTID_B32 : DS_Real_gfx10<0x0b0>;
1230defm DS_READ_ADDTID_B32  : DS_Real_gfx10<0x0b1>;
1231
1232//===----------------------------------------------------------------------===//
1233// GFX10, GFX11.
1234//===----------------------------------------------------------------------===//
1235
1236multiclass DS_Real_gfx10_gfx11<bits<8> op> :
1237  DS_Real_gfx10<op>, DS_Real_gfx11<op>;
1238
1239defm DS_ADD_F32          : DS_Real_gfx10_gfx11<0x015>;
1240defm DS_ADD_SRC2_F32     : DS_Real_gfx10<0x095>;
1241defm DS_PERMUTE_B32      : DS_Real_gfx10_gfx11<0x0b2>;
1242defm DS_BPERMUTE_B32     : DS_Real_gfx10_gfx11<0x0b3>;
1243
1244//===----------------------------------------------------------------------===//
1245// GFX7, GFX10, GFX11.
1246//===----------------------------------------------------------------------===//
1247
1248let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in {
1249  multiclass DS_Real_gfx7<bits<8> op> {
1250    def _gfx7 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, !cast<DS_Pseudo>(NAME),
1251                                             SIEncodingFamily.SI>;
1252  }
1253} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7"
1254
1255multiclass DS_Real_gfx7_gfx10_gfx11<bits<8> op> :
1256  DS_Real_gfx7<op>, DS_Real_gfx10_gfx11<op>;
1257
1258multiclass DS_Real_gfx7_gfx10<bits<8> op> :
1259  DS_Real_gfx7<op>, DS_Real_gfx10<op>;
1260
1261// FIXME-GFX7: Add tests when upstreaming this part.
1262defm DS_GWS_SEMA_RELEASE_ALL : DS_Real_gfx7_gfx10_gfx11<0x018>;
1263defm DS_WRAP_RTN_B32         : DS_Real_gfx7_gfx10_gfx11<0x034>;
1264defm DS_CONDXCHG32_RTN_B64   : DS_Real_gfx7_gfx10_gfx11<0x07e>;
1265defm DS_WRITE_B96            : DS_Real_gfx7_gfx10<0x0de>;
1266defm DS_WRITE_B128           : DS_Real_gfx7_gfx10<0x0df>;
1267defm DS_READ_B96             : DS_Real_gfx7_gfx10<0x0fe>;
1268defm DS_READ_B128            : DS_Real_gfx7_gfx10<0x0ff>;
1269
1270//===----------------------------------------------------------------------===//
1271// GFX6, GFX7, GFX10, GFX11.
1272//===----------------------------------------------------------------------===//
1273
1274let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {
1275  multiclass DS_Real_gfx6_gfx7<bits<8> op> {
1276    def _gfx6_gfx7 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, !cast<DS_Pseudo>(NAME),
1277                                                  SIEncodingFamily.SI>;
1278  }
1279} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"
1280
1281multiclass DS_Real_gfx6_gfx7_gfx10_gfx11<bits<8> op> :
1282  DS_Real_gfx6_gfx7<op>, DS_Real_gfx10_gfx11<op>;
1283
1284multiclass DS_Real_gfx6_gfx7_gfx10<bits<8> op> :
1285  DS_Real_gfx6_gfx7<op>, DS_Real_gfx10<op>;
1286
1287defm DS_ADD_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x000>;
1288defm DS_SUB_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x001>;
1289defm DS_RSUB_U32            : DS_Real_gfx6_gfx7_gfx10_gfx11<0x002>;
1290defm DS_INC_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x003>;
1291defm DS_DEC_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x004>;
1292defm DS_MIN_I32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x005>;
1293defm DS_MAX_I32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x006>;
1294defm DS_MIN_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x007>;
1295defm DS_MAX_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x008>;
1296defm DS_AND_B32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x009>;
1297defm DS_OR_B32              : DS_Real_gfx6_gfx7_gfx10_gfx11<0x00a>;
1298defm DS_XOR_B32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x00b>;
1299defm DS_MSKOR_B32           : DS_Real_gfx6_gfx7_gfx10_gfx11<0x00c>;
1300
1301defm DS_WRITE_B32           : DS_Real_gfx6_gfx7_gfx10<0x00d>;
1302defm DS_WRITE2_B32          : DS_Real_gfx6_gfx7_gfx10<0x00e>;
1303defm DS_WRITE2ST64_B32      : DS_Real_gfx6_gfx7_gfx10<0x00f>;
1304defm DS_CMPST_B32           : DS_Real_gfx6_gfx7_gfx10<0x010>;
1305defm DS_CMPST_F32           : DS_Real_gfx6_gfx7_gfx10<0x011>;
1306
1307defm DS_MIN_F32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x012>;
1308defm DS_MAX_F32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x013>;
1309defm DS_NOP                 : DS_Real_gfx6_gfx7_gfx10_gfx11<0x014>;
1310defm DS_GWS_INIT            : DS_Real_gfx6_gfx7_gfx10_gfx11<0x019>;
1311defm DS_GWS_SEMA_V          : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01a>;
1312defm DS_GWS_SEMA_BR         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01b>;
1313defm DS_GWS_SEMA_P          : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01c>;
1314defm DS_GWS_BARRIER         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01d>;
1315
1316defm DS_WRITE_B8            : DS_Real_gfx6_gfx7_gfx10<0x01e>;
1317defm DS_WRITE_B16           : DS_Real_gfx6_gfx7_gfx10<0x01f>;
1318
1319defm DS_ADD_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x020>;
1320defm DS_SUB_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x021>;
1321defm DS_RSUB_RTN_U32        : DS_Real_gfx6_gfx7_gfx10_gfx11<0x022>;
1322defm DS_INC_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x023>;
1323defm DS_DEC_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x024>;
1324defm DS_MIN_RTN_I32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x025>;
1325defm DS_MAX_RTN_I32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x026>;
1326defm DS_MIN_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x027>;
1327defm DS_MAX_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x028>;
1328defm DS_AND_RTN_B32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x029>;
1329defm DS_OR_RTN_B32          : DS_Real_gfx6_gfx7_gfx10_gfx11<0x02a>;
1330defm DS_XOR_RTN_B32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x02b>;
1331defm DS_MSKOR_RTN_B32       : DS_Real_gfx6_gfx7_gfx10_gfx11<0x02c>;
1332
1333defm DS_WRXCHG_RTN_B32      : DS_Real_gfx6_gfx7_gfx10<0x02d>;
1334defm DS_WRXCHG2_RTN_B32     : DS_Real_gfx6_gfx7_gfx10<0x02e>;
1335defm DS_WRXCHG2ST64_RTN_B32 : DS_Real_gfx6_gfx7_gfx10<0x02f>;
1336defm DS_CMPST_RTN_B32       : DS_Real_gfx6_gfx7_gfx10<0x030>;
1337defm DS_CMPST_RTN_F32       : DS_Real_gfx6_gfx7_gfx10<0x031>;
1338
1339defm DS_MIN_RTN_F32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x032>;
1340defm DS_MAX_RTN_F32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x033>;
1341defm DS_SWIZZLE_B32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x035>;
1342
1343defm DS_READ_B32            : DS_Real_gfx6_gfx7_gfx10<0x036>;
1344defm DS_READ2_B32           : DS_Real_gfx6_gfx7_gfx10<0x037>;
1345defm DS_READ2ST64_B32       : DS_Real_gfx6_gfx7_gfx10<0x038>;
1346defm DS_READ_I8             : DS_Real_gfx6_gfx7_gfx10<0x039>;
1347defm DS_READ_U8             : DS_Real_gfx6_gfx7_gfx10<0x03a>;
1348defm DS_READ_I16            : DS_Real_gfx6_gfx7_gfx10<0x03b>;
1349defm DS_READ_U16            : DS_Real_gfx6_gfx7_gfx10<0x03c>;
1350
1351defm DS_CONSUME             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x03d>;
1352defm DS_APPEND              : DS_Real_gfx6_gfx7_gfx10_gfx11<0x03e>;
1353defm DS_ORDERED_COUNT       : DS_Real_gfx6_gfx7_gfx10_gfx11<0x03f>;
1354defm DS_ADD_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x040>;
1355defm DS_SUB_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x041>;
1356defm DS_RSUB_U64            : DS_Real_gfx6_gfx7_gfx10_gfx11<0x042>;
1357defm DS_INC_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x043>;
1358defm DS_DEC_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x044>;
1359defm DS_MIN_I64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x045>;
1360defm DS_MAX_I64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x046>;
1361defm DS_MIN_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x047>;
1362defm DS_MAX_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x048>;
1363defm DS_AND_B64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x049>;
1364defm DS_OR_B64              : DS_Real_gfx6_gfx7_gfx10_gfx11<0x04a>;
1365defm DS_XOR_B64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x04b>;
1366defm DS_MSKOR_B64           : DS_Real_gfx6_gfx7_gfx10_gfx11<0x04c>;
1367
1368defm DS_WRITE_B64           : DS_Real_gfx6_gfx7_gfx10<0x04d>;
1369defm DS_WRITE2_B64          : DS_Real_gfx6_gfx7_gfx10<0x04e>;
1370defm DS_WRITE2ST64_B64      : DS_Real_gfx6_gfx7_gfx10<0x04f>;
1371defm DS_CMPST_B64           : DS_Real_gfx6_gfx7_gfx10<0x050>;
1372defm DS_CMPST_F64           : DS_Real_gfx6_gfx7_gfx10<0x051>;
1373
1374defm DS_MIN_F64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x052>;
1375defm DS_MAX_F64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x053>;
1376defm DS_ADD_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x060>;
1377defm DS_SUB_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x061>;
1378defm DS_RSUB_RTN_U64        : DS_Real_gfx6_gfx7_gfx10_gfx11<0x062>;
1379defm DS_INC_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x063>;
1380defm DS_DEC_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x064>;
1381defm DS_MIN_RTN_I64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x065>;
1382defm DS_MAX_RTN_I64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x066>;
1383defm DS_MIN_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x067>;
1384defm DS_MAX_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x068>;
1385defm DS_AND_RTN_B64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x069>;
1386defm DS_OR_RTN_B64          : DS_Real_gfx6_gfx7_gfx10_gfx11<0x06a>;
1387defm DS_XOR_RTN_B64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x06b>;
1388defm DS_MSKOR_RTN_B64       : DS_Real_gfx6_gfx7_gfx10_gfx11<0x06c>;
1389
1390defm DS_WRXCHG_RTN_B64      : DS_Real_gfx6_gfx7_gfx10<0x06d>;
1391defm DS_WRXCHG2_RTN_B64     : DS_Real_gfx6_gfx7_gfx10<0x06e>;
1392defm DS_WRXCHG2ST64_RTN_B64 : DS_Real_gfx6_gfx7_gfx10<0x06f>;
1393defm DS_CMPST_RTN_B64       : DS_Real_gfx6_gfx7_gfx10<0x070>;
1394defm DS_CMPST_RTN_F64       : DS_Real_gfx6_gfx7_gfx10<0x071>;
1395
1396defm DS_MIN_RTN_F64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x072>;
1397defm DS_MAX_RTN_F64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x073>;
1398
1399defm DS_READ_B64            : DS_Real_gfx6_gfx7_gfx10<0x076>;
1400defm DS_READ2_B64           : DS_Real_gfx6_gfx7_gfx10<0x077>;
1401defm DS_READ2ST64_B64       : DS_Real_gfx6_gfx7_gfx10<0x078>;
1402defm DS_ADD_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x080>;
1403defm DS_SUB_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x081>;
1404defm DS_RSUB_SRC2_U32       : DS_Real_gfx6_gfx7_gfx10<0x082>;
1405defm DS_INC_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x083>;
1406defm DS_DEC_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x084>;
1407defm DS_MIN_SRC2_I32        : DS_Real_gfx6_gfx7_gfx10<0x085>;
1408defm DS_MAX_SRC2_I32        : DS_Real_gfx6_gfx7_gfx10<0x086>;
1409defm DS_MIN_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x087>;
1410defm DS_MAX_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x088>;
1411defm DS_AND_SRC2_B32        : DS_Real_gfx6_gfx7_gfx10<0x089>;
1412defm DS_OR_SRC2_B32         : DS_Real_gfx6_gfx7_gfx10<0x08a>;
1413defm DS_XOR_SRC2_B32        : DS_Real_gfx6_gfx7_gfx10<0x08b>;
1414defm DS_WRITE_SRC2_B32      : DS_Real_gfx6_gfx7_gfx10<0x08d>;
1415defm DS_MIN_SRC2_F32        : DS_Real_gfx6_gfx7_gfx10<0x092>;
1416defm DS_MAX_SRC2_F32        : DS_Real_gfx6_gfx7_gfx10<0x093>;
1417defm DS_ADD_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c0>;
1418defm DS_SUB_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c1>;
1419defm DS_RSUB_SRC2_U64       : DS_Real_gfx6_gfx7_gfx10<0x0c2>;
1420defm DS_INC_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c3>;
1421defm DS_DEC_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c4>;
1422defm DS_MIN_SRC2_I64        : DS_Real_gfx6_gfx7_gfx10<0x0c5>;
1423defm DS_MAX_SRC2_I64        : DS_Real_gfx6_gfx7_gfx10<0x0c6>;
1424defm DS_MIN_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c7>;
1425defm DS_MAX_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c8>;
1426defm DS_AND_SRC2_B64        : DS_Real_gfx6_gfx7_gfx10<0x0c9>;
1427defm DS_OR_SRC2_B64         : DS_Real_gfx6_gfx7_gfx10<0x0ca>;
1428defm DS_XOR_SRC2_B64        : DS_Real_gfx6_gfx7_gfx10<0x0cb>;
1429defm DS_WRITE_SRC2_B64      : DS_Real_gfx6_gfx7_gfx10<0x0cd>;
1430defm DS_MIN_SRC2_F64        : DS_Real_gfx6_gfx7_gfx10<0x0d2>;
1431defm DS_MAX_SRC2_F64        : DS_Real_gfx6_gfx7_gfx10<0x0d3>;
1432
1433//===----------------------------------------------------------------------===//
1434// GFX8, GFX9 (VI).
1435//===----------------------------------------------------------------------===//
1436
1437class DS_Real_vi <bits<8> op, DS_Pseudo ps> :
1438  DS_Real <ps>,
1439  SIMCInstr <ps.Mnemonic, SIEncodingFamily.VI> {
1440  let AssemblerPredicate = isGFX8GFX9;
1441  let DecoderNamespace = "GFX8";
1442
1443  // encoding
1444  let Inst{7-0}   = !if(ps.has_offset0, offset0, 0);
1445  let Inst{15-8}  = !if(ps.has_offset1, offset1, 0);
1446  let Inst{16}    = !if(ps.has_gds, gds, ps.gdsValue);
1447  let Inst{24-17} = op;
1448  let Inst{25}    = acc;
1449  let Inst{31-26} = 0x36; // ds prefix
1450  let Inst{39-32} = !if(ps.has_addr, addr, !if(ps.has_gws_data0, data0{7-0}, 0));
1451  let Inst{47-40} = !if(ps.has_data0, data0{7-0}, 0);
1452  let Inst{55-48} = !if(ps.has_data1, data1{7-0}, 0);
1453  let Inst{63-56} = !if(ps.has_vdst, vdst{7-0}, 0);
1454}
1455
1456def DS_ADD_U32_vi         : DS_Real_vi<0x0,  DS_ADD_U32>;
1457def DS_SUB_U32_vi         : DS_Real_vi<0x1,  DS_SUB_U32>;
1458def DS_RSUB_U32_vi        : DS_Real_vi<0x2,  DS_RSUB_U32>;
1459def DS_INC_U32_vi         : DS_Real_vi<0x3,  DS_INC_U32>;
1460def DS_DEC_U32_vi         : DS_Real_vi<0x4,  DS_DEC_U32>;
1461def DS_MIN_I32_vi         : DS_Real_vi<0x5,  DS_MIN_I32>;
1462def DS_MAX_I32_vi         : DS_Real_vi<0x6,  DS_MAX_I32>;
1463def DS_MIN_U32_vi         : DS_Real_vi<0x7,  DS_MIN_U32>;
1464def DS_MAX_U32_vi         : DS_Real_vi<0x8,  DS_MAX_U32>;
1465def DS_AND_B32_vi         : DS_Real_vi<0x9,  DS_AND_B32>;
1466def DS_OR_B32_vi          : DS_Real_vi<0xa,  DS_OR_B32>;
1467def DS_XOR_B32_vi         : DS_Real_vi<0xb,  DS_XOR_B32>;
1468def DS_MSKOR_B32_vi       : DS_Real_vi<0xc,  DS_MSKOR_B32>;
1469def DS_WRITE_B32_vi       : DS_Real_vi<0xd,  DS_WRITE_B32>;
1470def DS_WRITE2_B32_vi      : DS_Real_vi<0xe,  DS_WRITE2_B32>;
1471def DS_WRITE2ST64_B32_vi  : DS_Real_vi<0xf,  DS_WRITE2ST64_B32>;
1472def DS_CMPST_B32_vi       : DS_Real_vi<0x10, DS_CMPST_B32>;
1473def DS_CMPST_F32_vi       : DS_Real_vi<0x11, DS_CMPST_F32>;
1474def DS_MIN_F32_vi         : DS_Real_vi<0x12, DS_MIN_F32>;
1475def DS_MAX_F32_vi         : DS_Real_vi<0x13, DS_MAX_F32>;
1476def DS_NOP_vi             : DS_Real_vi<0x14, DS_NOP>;
1477def DS_ADD_F32_vi         : DS_Real_vi<0x15, DS_ADD_F32>;
1478def DS_GWS_INIT_vi        : DS_Real_vi<0x99, DS_GWS_INIT>;
1479def DS_GWS_SEMA_V_vi      : DS_Real_vi<0x9a, DS_GWS_SEMA_V>;
1480def DS_GWS_SEMA_BR_vi     : DS_Real_vi<0x9b, DS_GWS_SEMA_BR>;
1481def DS_GWS_SEMA_P_vi      : DS_Real_vi<0x9c, DS_GWS_SEMA_P>;
1482def DS_GWS_BARRIER_vi     : DS_Real_vi<0x9d, DS_GWS_BARRIER>;
1483def DS_WRITE_ADDTID_B32_vi : DS_Real_vi<0x1d, DS_WRITE_ADDTID_B32>;
1484def DS_WRITE_B8_vi        : DS_Real_vi<0x1e, DS_WRITE_B8>;
1485def DS_WRITE_B16_vi       : DS_Real_vi<0x1f, DS_WRITE_B16>;
1486def DS_ADD_RTN_U32_vi     : DS_Real_vi<0x20, DS_ADD_RTN_U32>;
1487def DS_SUB_RTN_U32_vi     : DS_Real_vi<0x21, DS_SUB_RTN_U32>;
1488def DS_RSUB_RTN_U32_vi    : DS_Real_vi<0x22, DS_RSUB_RTN_U32>;
1489def DS_INC_RTN_U32_vi     : DS_Real_vi<0x23, DS_INC_RTN_U32>;
1490def DS_DEC_RTN_U32_vi     : DS_Real_vi<0x24, DS_DEC_RTN_U32>;
1491def DS_MIN_RTN_I32_vi     : DS_Real_vi<0x25, DS_MIN_RTN_I32>;
1492def DS_MAX_RTN_I32_vi     : DS_Real_vi<0x26, DS_MAX_RTN_I32>;
1493def DS_MIN_RTN_U32_vi     : DS_Real_vi<0x27, DS_MIN_RTN_U32>;
1494def DS_MAX_RTN_U32_vi     : DS_Real_vi<0x28, DS_MAX_RTN_U32>;
1495def DS_AND_RTN_B32_vi     : DS_Real_vi<0x29, DS_AND_RTN_B32>;
1496def DS_OR_RTN_B32_vi      : DS_Real_vi<0x2a, DS_OR_RTN_B32>;
1497def DS_XOR_RTN_B32_vi     : DS_Real_vi<0x2b, DS_XOR_RTN_B32>;
1498def DS_MSKOR_RTN_B32_vi   : DS_Real_vi<0x2c, DS_MSKOR_RTN_B32>;
1499def DS_WRXCHG_RTN_B32_vi  : DS_Real_vi<0x2d, DS_WRXCHG_RTN_B32>;
1500def DS_WRXCHG2_RTN_B32_vi : DS_Real_vi<0x2e, DS_WRXCHG2_RTN_B32>;
1501def DS_WRXCHG2ST64_RTN_B32_vi : DS_Real_vi<0x2f, DS_WRXCHG2ST64_RTN_B32>;
1502def DS_CMPST_RTN_B32_vi   : DS_Real_vi<0x30, DS_CMPST_RTN_B32>;
1503def DS_CMPST_RTN_F32_vi   : DS_Real_vi<0x31, DS_CMPST_RTN_F32>;
1504def DS_MIN_RTN_F32_vi     : DS_Real_vi<0x32, DS_MIN_RTN_F32>;
1505def DS_MAX_RTN_F32_vi     : DS_Real_vi<0x33, DS_MAX_RTN_F32>;
1506def DS_WRAP_RTN_B32_vi    : DS_Real_vi<0x34, DS_WRAP_RTN_B32>;
1507def DS_ADD_RTN_F32_vi     : DS_Real_vi<0x35, DS_ADD_RTN_F32>;
1508def DS_READ_B32_vi        : DS_Real_vi<0x36, DS_READ_B32>;
1509def DS_READ2_B32_vi       : DS_Real_vi<0x37, DS_READ2_B32>;
1510def DS_READ2ST64_B32_vi   : DS_Real_vi<0x38, DS_READ2ST64_B32>;
1511def DS_READ_I8_vi         : DS_Real_vi<0x39, DS_READ_I8>;
1512def DS_READ_U8_vi         : DS_Real_vi<0x3a, DS_READ_U8>;
1513def DS_READ_I16_vi        : DS_Real_vi<0x3b, DS_READ_I16>;
1514def DS_READ_U16_vi        : DS_Real_vi<0x3c, DS_READ_U16>;
1515def DS_READ_ADDTID_B32_vi : DS_Real_vi<0xb6, DS_READ_ADDTID_B32>;
1516def DS_CONSUME_vi         : DS_Real_vi<0xbd, DS_CONSUME>;
1517def DS_APPEND_vi          : DS_Real_vi<0xbe, DS_APPEND>;
1518def DS_ORDERED_COUNT_vi   : DS_Real_vi<0xbf, DS_ORDERED_COUNT>;
1519def DS_SWIZZLE_B32_vi     : DS_Real_vi<0x3d, DS_SWIZZLE_B32>;
1520def DS_PERMUTE_B32_vi     : DS_Real_vi<0x3e, DS_PERMUTE_B32>;
1521def DS_BPERMUTE_B32_vi    : DS_Real_vi<0x3f, DS_BPERMUTE_B32>;
1522
1523def DS_ADD_U64_vi         : DS_Real_vi<0x40, DS_ADD_U64>;
1524def DS_SUB_U64_vi         : DS_Real_vi<0x41, DS_SUB_U64>;
1525def DS_RSUB_U64_vi        : DS_Real_vi<0x42, DS_RSUB_U64>;
1526def DS_INC_U64_vi         : DS_Real_vi<0x43, DS_INC_U64>;
1527def DS_DEC_U64_vi         : DS_Real_vi<0x44, DS_DEC_U64>;
1528def DS_MIN_I64_vi         : DS_Real_vi<0x45, DS_MIN_I64>;
1529def DS_MAX_I64_vi         : DS_Real_vi<0x46, DS_MAX_I64>;
1530def DS_MIN_U64_vi         : DS_Real_vi<0x47, DS_MIN_U64>;
1531def DS_MAX_U64_vi         : DS_Real_vi<0x48, DS_MAX_U64>;
1532def DS_AND_B64_vi         : DS_Real_vi<0x49, DS_AND_B64>;
1533def DS_OR_B64_vi          : DS_Real_vi<0x4a, DS_OR_B64>;
1534def DS_XOR_B64_vi         : DS_Real_vi<0x4b, DS_XOR_B64>;
1535def DS_MSKOR_B64_vi       : DS_Real_vi<0x4c, DS_MSKOR_B64>;
1536def DS_WRITE_B64_vi       : DS_Real_vi<0x4d, DS_WRITE_B64>;
1537def DS_WRITE2_B64_vi      : DS_Real_vi<0x4E, DS_WRITE2_B64>;
1538def DS_WRITE2ST64_B64_vi  : DS_Real_vi<0x4f, DS_WRITE2ST64_B64>;
1539def DS_CMPST_B64_vi       : DS_Real_vi<0x50, DS_CMPST_B64>;
1540def DS_CMPST_F64_vi       : DS_Real_vi<0x51, DS_CMPST_F64>;
1541def DS_MIN_F64_vi         : DS_Real_vi<0x52, DS_MIN_F64>;
1542def DS_MAX_F64_vi         : DS_Real_vi<0x53, DS_MAX_F64>;
1543
1544def DS_WRITE_B8_D16_HI_vi  : DS_Real_vi<0x54, DS_WRITE_B8_D16_HI>;
1545def DS_WRITE_B16_D16_HI_vi : DS_Real_vi<0x55, DS_WRITE_B16_D16_HI>;
1546
1547def DS_READ_U8_D16_vi     : DS_Real_vi<0x56, DS_READ_U8_D16>;
1548def DS_READ_U8_D16_HI_vi  : DS_Real_vi<0x57, DS_READ_U8_D16_HI>;
1549def DS_READ_I8_D16_vi     : DS_Real_vi<0x58, DS_READ_I8_D16>;
1550def DS_READ_I8_D16_HI_vi  : DS_Real_vi<0x59, DS_READ_I8_D16_HI>;
1551def DS_READ_U16_D16_vi    : DS_Real_vi<0x5a, DS_READ_U16_D16>;
1552def DS_READ_U16_D16_HI_vi : DS_Real_vi<0x5b, DS_READ_U16_D16_HI>;
1553
1554def DS_ADD_RTN_U64_vi     : DS_Real_vi<0x60, DS_ADD_RTN_U64>;
1555def DS_SUB_RTN_U64_vi     : DS_Real_vi<0x61, DS_SUB_RTN_U64>;
1556def DS_RSUB_RTN_U64_vi    : DS_Real_vi<0x62, DS_RSUB_RTN_U64>;
1557def DS_INC_RTN_U64_vi     : DS_Real_vi<0x63, DS_INC_RTN_U64>;
1558def DS_DEC_RTN_U64_vi     : DS_Real_vi<0x64, DS_DEC_RTN_U64>;
1559def DS_MIN_RTN_I64_vi     : DS_Real_vi<0x65, DS_MIN_RTN_I64>;
1560def DS_MAX_RTN_I64_vi     : DS_Real_vi<0x66, DS_MAX_RTN_I64>;
1561def DS_MIN_RTN_U64_vi     : DS_Real_vi<0x67, DS_MIN_RTN_U64>;
1562def DS_MAX_RTN_U64_vi     : DS_Real_vi<0x68, DS_MAX_RTN_U64>;
1563def DS_AND_RTN_B64_vi     : DS_Real_vi<0x69, DS_AND_RTN_B64>;
1564def DS_OR_RTN_B64_vi      : DS_Real_vi<0x6a, DS_OR_RTN_B64>;
1565def DS_XOR_RTN_B64_vi     : DS_Real_vi<0x6b, DS_XOR_RTN_B64>;
1566def DS_MSKOR_RTN_B64_vi   : DS_Real_vi<0x6c, DS_MSKOR_RTN_B64>;
1567def DS_WRXCHG_RTN_B64_vi  : DS_Real_vi<0x6d, DS_WRXCHG_RTN_B64>;
1568def DS_WRXCHG2_RTN_B64_vi : DS_Real_vi<0x6e, DS_WRXCHG2_RTN_B64>;
1569def DS_WRXCHG2ST64_RTN_B64_vi : DS_Real_vi<0x6f, DS_WRXCHG2ST64_RTN_B64>;
1570def DS_CONDXCHG32_RTN_B64_vi   : DS_Real_vi<0x7e, DS_CONDXCHG32_RTN_B64>;
1571def DS_GWS_SEMA_RELEASE_ALL_vi : DS_Real_vi<0x98, DS_GWS_SEMA_RELEASE_ALL>;
1572def DS_CMPST_RTN_B64_vi   : DS_Real_vi<0x70, DS_CMPST_RTN_B64>;
1573def DS_CMPST_RTN_F64_vi   : DS_Real_vi<0x71, DS_CMPST_RTN_F64>;
1574def DS_MIN_RTN_F64_vi     : DS_Real_vi<0x72, DS_MIN_RTN_F64>;
1575def DS_MAX_RTN_F64_vi     : DS_Real_vi<0x73, DS_MAX_RTN_F64>;
1576
1577def DS_READ_B64_vi        : DS_Real_vi<0x76, DS_READ_B64>;
1578def DS_READ2_B64_vi       : DS_Real_vi<0x77, DS_READ2_B64>;
1579def DS_READ2ST64_B64_vi   : DS_Real_vi<0x78, DS_READ2ST64_B64>;
1580
1581def DS_ADD_SRC2_U32_vi    : DS_Real_vi<0x80, DS_ADD_SRC2_U32>;
1582def DS_SUB_SRC2_U32_vi    : DS_Real_vi<0x81, DS_SUB_SRC2_U32>;
1583def DS_RSUB_SRC2_U32_vi   : DS_Real_vi<0x82, DS_RSUB_SRC2_U32>;
1584def DS_INC_SRC2_U32_vi    : DS_Real_vi<0x83, DS_INC_SRC2_U32>;
1585def DS_DEC_SRC2_U32_vi    : DS_Real_vi<0x84, DS_DEC_SRC2_U32>;
1586def DS_MIN_SRC2_I32_vi    : DS_Real_vi<0x85, DS_MIN_SRC2_I32>;
1587def DS_MAX_SRC2_I32_vi    : DS_Real_vi<0x86, DS_MAX_SRC2_I32>;
1588def DS_MIN_SRC2_U32_vi    : DS_Real_vi<0x87, DS_MIN_SRC2_U32>;
1589def DS_MAX_SRC2_U32_vi    : DS_Real_vi<0x88, DS_MAX_SRC2_U32>;
1590def DS_AND_SRC2_B32_vi    : DS_Real_vi<0x89, DS_AND_SRC2_B32>;
1591def DS_OR_SRC2_B32_vi     : DS_Real_vi<0x8a, DS_OR_SRC2_B32>;
1592def DS_XOR_SRC2_B32_vi    : DS_Real_vi<0x8b, DS_XOR_SRC2_B32>;
1593def DS_WRITE_SRC2_B32_vi  : DS_Real_vi<0x8d, DS_WRITE_SRC2_B32>;
1594def DS_MIN_SRC2_F32_vi    : DS_Real_vi<0x92, DS_MIN_SRC2_F32>;
1595def DS_MAX_SRC2_F32_vi    : DS_Real_vi<0x93, DS_MAX_SRC2_F32>;
1596def DS_ADD_SRC2_F32_vi    : DS_Real_vi<0x95, DS_ADD_SRC2_F32>;
1597def DS_ADD_SRC2_U64_vi    : DS_Real_vi<0xc0, DS_ADD_SRC2_U64>;
1598def DS_SUB_SRC2_U64_vi    : DS_Real_vi<0xc1, DS_SUB_SRC2_U64>;
1599def DS_RSUB_SRC2_U64_vi   : DS_Real_vi<0xc2, DS_RSUB_SRC2_U64>;
1600def DS_INC_SRC2_U64_vi    : DS_Real_vi<0xc3, DS_INC_SRC2_U64>;
1601def DS_DEC_SRC2_U64_vi    : DS_Real_vi<0xc4, DS_DEC_SRC2_U64>;
1602def DS_MIN_SRC2_I64_vi    : DS_Real_vi<0xc5, DS_MIN_SRC2_I64>;
1603def DS_MAX_SRC2_I64_vi    : DS_Real_vi<0xc6, DS_MAX_SRC2_I64>;
1604def DS_MIN_SRC2_U64_vi    : DS_Real_vi<0xc7, DS_MIN_SRC2_U64>;
1605def DS_MAX_SRC2_U64_vi    : DS_Real_vi<0xc8, DS_MAX_SRC2_U64>;
1606def DS_AND_SRC2_B64_vi    : DS_Real_vi<0xc9, DS_AND_SRC2_B64>;
1607def DS_OR_SRC2_B64_vi     : DS_Real_vi<0xca, DS_OR_SRC2_B64>;
1608def DS_XOR_SRC2_B64_vi    : DS_Real_vi<0xcb, DS_XOR_SRC2_B64>;
1609def DS_WRITE_SRC2_B64_vi  : DS_Real_vi<0xcd, DS_WRITE_SRC2_B64>;
1610def DS_MIN_SRC2_F64_vi    : DS_Real_vi<0xd2, DS_MIN_SRC2_F64>;
1611def DS_MAX_SRC2_F64_vi    : DS_Real_vi<0xd3, DS_MAX_SRC2_F64>;
1612def DS_WRITE_B96_vi       : DS_Real_vi<0xde, DS_WRITE_B96>;
1613def DS_WRITE_B128_vi      : DS_Real_vi<0xdf, DS_WRITE_B128>;
1614def DS_READ_B96_vi        : DS_Real_vi<0xfe, DS_READ_B96>;
1615def DS_READ_B128_vi       : DS_Real_vi<0xff, DS_READ_B128>;
1616
1617let SubtargetPredicate = isGFX90APlus in {
1618  def DS_ADD_F64_vi     : DS_Real_vi<0x5c, DS_ADD_F64>;
1619  def DS_ADD_RTN_F64_vi : DS_Real_vi<0x7c, DS_ADD_RTN_F64>;
1620} // End SubtargetPredicate = isGFX90APlus
1621
1622let SubtargetPredicate = isGFX940Plus in {
1623  def DS_PK_ADD_F16_vi     : DS_Real_vi<0x17, DS_PK_ADD_F16>;
1624  def DS_PK_ADD_RTN_F16_vi : DS_Real_vi<0xb7, DS_PK_ADD_RTN_F16>;
1625  def DS_PK_ADD_BF16_vi     : DS_Real_vi<0x18, DS_PK_ADD_BF16>;
1626  def DS_PK_ADD_RTN_BF16_vi : DS_Real_vi<0xb8, DS_PK_ADD_RTN_BF16>;
1627} // End SubtargetPredicate = isGFX940Plus
1628