1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX7LESS %s
3; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX8 %s
4; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s
5; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX1064 %s
6; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX1032 %s
7
8declare i32 @llvm.amdgcn.workitem.id.x()
9
10@local_var32 = addrspace(3) global i32 undef, align 4
11@local_var64 = addrspace(3) global i64 undef, align 8
12
13; Show what the atomic optimization pass will do for local pointers.
14
15define amdgpu_kernel void @add_i32_constant(i32 addrspace(1)* %out) {
16;
17;
18; GFX7LESS-LABEL: add_i32_constant:
19; GFX7LESS:       ; %bb.0: ; %entry
20; GFX7LESS-NEXT:    s_mov_b64 s[2:3], exec
21; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
22; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
23; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
24; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
25; GFX7LESS-NEXT:    ; implicit-def: $vgpr1
26; GFX7LESS-NEXT:    s_and_saveexec_b64 s[4:5], vcc
27; GFX7LESS-NEXT:    s_cbranch_execz BB0_2
28; GFX7LESS-NEXT:  ; %bb.1:
29; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
30; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
31; GFX7LESS-NEXT:    v_mul_u32_u24_e64 v2, s2, 5
32; GFX7LESS-NEXT:    s_mov_b32 m0, -1
33; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
34; GFX7LESS-NEXT:    ds_add_rtn_u32 v1, v1, v2
35; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
36; GFX7LESS-NEXT:    buffer_wbinvl1
37; GFX7LESS-NEXT:  BB0_2:
38; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[4:5]
39; GFX7LESS-NEXT:    v_readfirstlane_b32 s2, v1
40; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
41; GFX7LESS-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
42; GFX7LESS-NEXT:    s_mov_b32 s2, -1
43; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
44; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
45; GFX7LESS-NEXT:    s_endpgm
46;
47; GFX8-LABEL: add_i32_constant:
48; GFX8:       ; %bb.0: ; %entry
49; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
50; GFX8-NEXT:    s_mov_b64 s[2:3], exec
51; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
52; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
53; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
54; GFX8-NEXT:    ; implicit-def: $vgpr1
55; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
56; GFX8-NEXT:    s_cbranch_execz BB0_2
57; GFX8-NEXT:  ; %bb.1:
58; GFX8-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
59; GFX8-NEXT:    v_mul_u32_u24_e64 v1, s2, 5
60; GFX8-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
61; GFX8-NEXT:    s_mov_b32 m0, -1
62; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
63; GFX8-NEXT:    ds_add_rtn_u32 v1, v2, v1
64; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
65; GFX8-NEXT:    buffer_wbinvl1_vol
66; GFX8-NEXT:  BB0_2:
67; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
68; GFX8-NEXT:    v_readfirstlane_b32 s2, v1
69; GFX8-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
70; GFX8-NEXT:    s_mov_b32 s3, 0xf000
71; GFX8-NEXT:    s_mov_b32 s2, -1
72; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
73; GFX8-NEXT:    s_nop 0
74; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
75; GFX8-NEXT:    s_endpgm
76;
77; GFX9-LABEL: add_i32_constant:
78; GFX9:       ; %bb.0: ; %entry
79; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
80; GFX9-NEXT:    s_mov_b64 s[2:3], exec
81; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
82; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
83; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
84; GFX9-NEXT:    ; implicit-def: $vgpr1
85; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
86; GFX9-NEXT:    s_cbranch_execz BB0_2
87; GFX9-NEXT:  ; %bb.1:
88; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
89; GFX9-NEXT:    v_mul_u32_u24_e64 v1, s2, 5
90; GFX9-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
91; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
92; GFX9-NEXT:    ds_add_rtn_u32 v1, v2, v1
93; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
94; GFX9-NEXT:    buffer_wbinvl1_vol
95; GFX9-NEXT:  BB0_2:
96; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
97; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
98; GFX9-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
99; GFX9-NEXT:    s_mov_b32 s3, 0xf000
100; GFX9-NEXT:    s_mov_b32 s2, -1
101; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
102; GFX9-NEXT:    s_nop 0
103; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
104; GFX9-NEXT:    s_endpgm
105;
106; GFX1064-LABEL: add_i32_constant:
107; GFX1064:       ; %bb.0: ; %entry
108; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
109; GFX1064-NEXT:    s_mov_b64 s[2:3], exec
110; GFX1064-NEXT:    ; implicit-def: $vgpr1
111; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
112; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
113; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
114; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
115; GFX1064-NEXT:    s_cbranch_execz BB0_2
116; GFX1064-NEXT:  ; %bb.1:
117; GFX1064-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
118; GFX1064-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
119; GFX1064-NEXT:    v_mul_u32_u24_e64 v1, s2, 5
120; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
121; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
122; GFX1064-NEXT:    ds_add_rtn_u32 v1, v2, v1
123; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
124; GFX1064-NEXT:    buffer_gl0_inv
125; GFX1064-NEXT:    buffer_gl1_inv
126; GFX1064-NEXT:  BB0_2:
127; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
128; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
129; GFX1064-NEXT:    v_readfirstlane_b32 s2, v1
130; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
131; GFX1064-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
132; GFX1064-NEXT:    s_mov_b32 s2, -1
133; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
134; GFX1064-NEXT:    s_nop 0
135; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
136; GFX1064-NEXT:    s_endpgm
137;
138; GFX1032-LABEL: add_i32_constant:
139; GFX1032:       ; %bb.0: ; %entry
140; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
141; GFX1032-NEXT:    s_mov_b32 s2, exec_lo
142; GFX1032-NEXT:    ; implicit-def: $vgpr1
143; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
144; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
145; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
146; GFX1032-NEXT:    s_cbranch_execz BB0_2
147; GFX1032-NEXT:  ; %bb.1:
148; GFX1032-NEXT:    s_bcnt1_i32_b32 s2, s2
149; GFX1032-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
150; GFX1032-NEXT:    v_mul_u32_u24_e64 v1, s2, 5
151; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
152; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
153; GFX1032-NEXT:    ds_add_rtn_u32 v1, v2, v1
154; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
155; GFX1032-NEXT:    buffer_gl0_inv
156; GFX1032-NEXT:    buffer_gl1_inv
157; GFX1032-NEXT:  BB0_2:
158; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
159; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s3
160; GFX1032-NEXT:    v_readfirstlane_b32 s2, v1
161; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
162; GFX1032-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
163; GFX1032-NEXT:    s_mov_b32 s2, -1
164; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
165; GFX1032-NEXT:    s_nop 0
166; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
167; GFX1032-NEXT:    s_endpgm
168entry:
169  %old = atomicrmw add i32 addrspace(3)* @local_var32, i32 5 acq_rel
170  store i32 %old, i32 addrspace(1)* %out
171  ret void
172}
173
174define amdgpu_kernel void @add_i32_uniform(i32 addrspace(1)* %out, i32 %additive) {
175;
176;
177; GFX7LESS-LABEL: add_i32_uniform:
178; GFX7LESS:       ; %bb.0: ; %entry
179; GFX7LESS-NEXT:    s_mov_b64 s[6:7], exec
180; GFX7LESS-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
181; GFX7LESS-NEXT:    s_load_dword s2, s[0:1], 0xb
182; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
183; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s7, v0
184; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
185; GFX7LESS-NEXT:    ; implicit-def: $vgpr1
186; GFX7LESS-NEXT:    s_and_saveexec_b64 s[0:1], vcc
187; GFX7LESS-NEXT:    s_cbranch_execz BB1_2
188; GFX7LESS-NEXT:  ; %bb.1:
189; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s3, s[6:7]
190; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
191; GFX7LESS-NEXT:    s_mul_i32 s3, s2, s3
192; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
193; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s3
194; GFX7LESS-NEXT:    s_mov_b32 m0, -1
195; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
196; GFX7LESS-NEXT:    ds_add_rtn_u32 v1, v1, v2
197; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
198; GFX7LESS-NEXT:    buffer_wbinvl1
199; GFX7LESS-NEXT:  BB1_2:
200; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[0:1]
201; GFX7LESS-NEXT:    v_readfirstlane_b32 s0, v1
202; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
203; GFX7LESS-NEXT:    v_mul_lo_u32 v0, s2, v0
204; GFX7LESS-NEXT:    s_mov_b32 s7, 0xf000
205; GFX7LESS-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
206; GFX7LESS-NEXT:    s_mov_b32 s6, -1
207; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[4:7], 0
208; GFX7LESS-NEXT:    s_endpgm
209;
210; GFX8-LABEL: add_i32_uniform:
211; GFX8:       ; %bb.0: ; %entry
212; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
213; GFX8-NEXT:    s_load_dword s0, s[0:1], 0x2c
214; GFX8-NEXT:    s_mov_b64 s[2:3], exec
215; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
216; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
217; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
218; GFX8-NEXT:    ; implicit-def: $vgpr1
219; GFX8-NEXT:    s_and_saveexec_b64 s[6:7], vcc
220; GFX8-NEXT:    s_cbranch_execz BB1_2
221; GFX8-NEXT:  ; %bb.1:
222; GFX8-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
223; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
224; GFX8-NEXT:    s_mul_i32 s1, s0, s1
225; GFX8-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
226; GFX8-NEXT:    v_mov_b32_e32 v2, s1
227; GFX8-NEXT:    s_mov_b32 m0, -1
228; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
229; GFX8-NEXT:    ds_add_rtn_u32 v1, v1, v2
230; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
231; GFX8-NEXT:    buffer_wbinvl1_vol
232; GFX8-NEXT:  BB1_2:
233; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]
234; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
235; GFX8-NEXT:    v_mul_lo_u32 v0, s0, v0
236; GFX8-NEXT:    v_readfirstlane_b32 s0, v1
237; GFX8-NEXT:    s_mov_b32 s7, 0xf000
238; GFX8-NEXT:    s_mov_b32 s6, -1
239; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
240; GFX8-NEXT:    buffer_store_dword v0, off, s[4:7], 0
241; GFX8-NEXT:    s_endpgm
242;
243; GFX9-LABEL: add_i32_uniform:
244; GFX9:       ; %bb.0: ; %entry
245; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
246; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x2c
247; GFX9-NEXT:    s_mov_b64 s[2:3], exec
248; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
249; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
250; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
251; GFX9-NEXT:    ; implicit-def: $vgpr1
252; GFX9-NEXT:    s_and_saveexec_b64 s[6:7], vcc
253; GFX9-NEXT:    s_cbranch_execz BB1_2
254; GFX9-NEXT:  ; %bb.1:
255; GFX9-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
256; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
257; GFX9-NEXT:    s_mul_i32 s1, s0, s1
258; GFX9-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
259; GFX9-NEXT:    v_mov_b32_e32 v2, s1
260; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
261; GFX9-NEXT:    ds_add_rtn_u32 v1, v1, v2
262; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
263; GFX9-NEXT:    buffer_wbinvl1_vol
264; GFX9-NEXT:  BB1_2:
265; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]
266; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
267; GFX9-NEXT:    v_mul_lo_u32 v0, s0, v0
268; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
269; GFX9-NEXT:    s_mov_b32 s7, 0xf000
270; GFX9-NEXT:    s_mov_b32 s6, -1
271; GFX9-NEXT:    v_add_u32_e32 v0, s0, v0
272; GFX9-NEXT:    buffer_store_dword v0, off, s[4:7], 0
273; GFX9-NEXT:    s_endpgm
274;
275; GFX1064-LABEL: add_i32_uniform:
276; GFX1064:       ; %bb.0: ; %entry
277; GFX1064-NEXT:    s_clause 0x1
278; GFX1064-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
279; GFX1064-NEXT:    s_load_dword s0, s[0:1], 0x2c
280; GFX1064-NEXT:    s_mov_b64 s[2:3], exec
281; GFX1064-NEXT:    ; implicit-def: $vgpr1
282; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
283; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
284; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
285; GFX1064-NEXT:    s_and_saveexec_b64 s[6:7], vcc
286; GFX1064-NEXT:    s_cbranch_execz BB1_2
287; GFX1064-NEXT:  ; %bb.1:
288; GFX1064-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
289; GFX1064-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
290; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
291; GFX1064-NEXT:    s_mul_i32 s1, s0, s1
292; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
293; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
294; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
295; GFX1064-NEXT:    ds_add_rtn_u32 v1, v1, v2
296; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
297; GFX1064-NEXT:    buffer_gl0_inv
298; GFX1064-NEXT:    buffer_gl1_inv
299; GFX1064-NEXT:  BB1_2:
300; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
301; GFX1064-NEXT:    s_or_b64 exec, exec, s[6:7]
302; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
303; GFX1064-NEXT:    v_mul_lo_u32 v0, s0, v0
304; GFX1064-NEXT:    v_readfirstlane_b32 s0, v1
305; GFX1064-NEXT:    s_mov_b32 s7, 0x31016000
306; GFX1064-NEXT:    s_mov_b32 s6, -1
307; GFX1064-NEXT:    v_add_nc_u32_e32 v0, s0, v0
308; GFX1064-NEXT:    buffer_store_dword v0, off, s[4:7], 0
309; GFX1064-NEXT:    s_endpgm
310;
311; GFX1032-LABEL: add_i32_uniform:
312; GFX1032:       ; %bb.0: ; %entry
313; GFX1032-NEXT:    s_clause 0x1
314; GFX1032-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
315; GFX1032-NEXT:    s_load_dword s0, s[0:1], 0x2c
316; GFX1032-NEXT:    s_mov_b32 s2, exec_lo
317; GFX1032-NEXT:    ; implicit-def: $vgpr1
318; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
319; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
320; GFX1032-NEXT:    s_and_saveexec_b32 s1, vcc_lo
321; GFX1032-NEXT:    s_cbranch_execz BB1_2
322; GFX1032-NEXT:  ; %bb.1:
323; GFX1032-NEXT:    s_bcnt1_i32_b32 s2, s2
324; GFX1032-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
325; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
326; GFX1032-NEXT:    s_mul_i32 s2, s0, s2
327; GFX1032-NEXT:    v_mov_b32_e32 v2, s2
328; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
329; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
330; GFX1032-NEXT:    ds_add_rtn_u32 v1, v1, v2
331; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
332; GFX1032-NEXT:    buffer_gl0_inv
333; GFX1032-NEXT:    buffer_gl1_inv
334; GFX1032-NEXT:  BB1_2:
335; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
336; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s1
337; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
338; GFX1032-NEXT:    v_mul_lo_u32 v0, s0, v0
339; GFX1032-NEXT:    v_readfirstlane_b32 s0, v1
340; GFX1032-NEXT:    s_mov_b32 s7, 0x31016000
341; GFX1032-NEXT:    s_mov_b32 s6, -1
342; GFX1032-NEXT:    v_add_nc_u32_e32 v0, s0, v0
343; GFX1032-NEXT:    buffer_store_dword v0, off, s[4:7], 0
344; GFX1032-NEXT:    s_endpgm
345entry:
346  %old = atomicrmw add i32 addrspace(3)* @local_var32, i32 %additive acq_rel
347  store i32 %old, i32 addrspace(1)* %out
348  ret void
349}
350
351define amdgpu_kernel void @add_i32_varying(i32 addrspace(1)* %out) {
352;
353;
354; GFX7LESS-LABEL: add_i32_varying:
355; GFX7LESS:       ; %bb.0: ; %entry
356; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
357; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
358; GFX7LESS-NEXT:    s_mov_b32 m0, -1
359; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
360; GFX7LESS-NEXT:    ds_add_rtn_u32 v0, v1, v0
361; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
362; GFX7LESS-NEXT:    buffer_wbinvl1
363; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
364; GFX7LESS-NEXT:    s_mov_b32 s2, -1
365; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
366; GFX7LESS-NEXT:    s_endpgm
367;
368; GFX8-LABEL: add_i32_varying:
369; GFX8:       ; %bb.0: ; %entry
370; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
371; GFX8-NEXT:    v_mov_b32_e32 v2, v0
372; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
373; GFX8-NEXT:    v_mov_b32_e32 v1, 0
374; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
375; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
376; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
377; GFX8-NEXT:    s_not_b64 exec, exec
378; GFX8-NEXT:    v_mov_b32_e32 v2, 0
379; GFX8-NEXT:    s_not_b64 exec, exec
380; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
381; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
382; GFX8-NEXT:    s_nop 1
383; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
384; GFX8-NEXT:    s_nop 1
385; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
386; GFX8-NEXT:    s_nop 1
387; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
388; GFX8-NEXT:    s_nop 1
389; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
390; GFX8-NEXT:    s_nop 1
391; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
392; GFX8-NEXT:    v_readlane_b32 s4, v2, 63
393; GFX8-NEXT:    s_nop 0
394; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
395; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
396; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
397; GFX8-NEXT:    ; implicit-def: $vgpr0
398; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
399; GFX8-NEXT:    s_cbranch_execz BB2_2
400; GFX8-NEXT:  ; %bb.1:
401; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
402; GFX8-NEXT:    v_mov_b32_e32 v3, s4
403; GFX8-NEXT:    s_mov_b32 m0, -1
404; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
405; GFX8-NEXT:    ds_add_rtn_u32 v0, v0, v3
406; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
407; GFX8-NEXT:    buffer_wbinvl1_vol
408; GFX8-NEXT:  BB2_2:
409; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
410; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
411; GFX8-NEXT:    v_mov_b32_e32 v0, v1
412; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
413; GFX8-NEXT:    s_mov_b32 s3, 0xf000
414; GFX8-NEXT:    s_mov_b32 s2, -1
415; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
416; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
417; GFX8-NEXT:    s_endpgm
418;
419; GFX9-LABEL: add_i32_varying:
420; GFX9:       ; %bb.0: ; %entry
421; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
422; GFX9-NEXT:    v_mov_b32_e32 v2, v0
423; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
424; GFX9-NEXT:    v_mov_b32_e32 v1, 0
425; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
426; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
427; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
428; GFX9-NEXT:    s_not_b64 exec, exec
429; GFX9-NEXT:    v_mov_b32_e32 v2, 0
430; GFX9-NEXT:    s_not_b64 exec, exec
431; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
432; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
433; GFX9-NEXT:    s_nop 1
434; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
435; GFX9-NEXT:    s_nop 1
436; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
437; GFX9-NEXT:    s_nop 1
438; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
439; GFX9-NEXT:    s_nop 1
440; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
441; GFX9-NEXT:    s_nop 1
442; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
443; GFX9-NEXT:    v_readlane_b32 s4, v2, 63
444; GFX9-NEXT:    s_nop 0
445; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
446; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
447; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
448; GFX9-NEXT:    ; implicit-def: $vgpr0
449; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
450; GFX9-NEXT:    s_cbranch_execz BB2_2
451; GFX9-NEXT:  ; %bb.1:
452; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
453; GFX9-NEXT:    v_mov_b32_e32 v3, s4
454; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
455; GFX9-NEXT:    ds_add_rtn_u32 v0, v0, v3
456; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
457; GFX9-NEXT:    buffer_wbinvl1_vol
458; GFX9-NEXT:  BB2_2:
459; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
460; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
461; GFX9-NEXT:    v_mov_b32_e32 v0, v1
462; GFX9-NEXT:    v_add_u32_e32 v0, s2, v0
463; GFX9-NEXT:    s_mov_b32 s3, 0xf000
464; GFX9-NEXT:    s_mov_b32 s2, -1
465; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
466; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
467; GFX9-NEXT:    s_endpgm
468;
469; GFX1064-LABEL: add_i32_varying:
470; GFX1064:       ; %bb.0: ; %entry
471; GFX1064-NEXT:    v_mov_b32_e32 v1, v0
472; GFX1064-NEXT:    s_not_b64 exec, exec
473; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
474; GFX1064-NEXT:    s_not_b64 exec, exec
475; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
476; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
477; GFX1064-NEXT:    v_mov_b32_e32 v3, 0
478; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
479; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
480; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
481; GFX1064-NEXT:    v_mov_b32_e32 v2, v1
482; GFX1064-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
483; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
484; GFX1064-NEXT:    v_readlane_b32 s4, v1, 31
485; GFX1064-NEXT:    v_mov_b32_e32 v2, s4
486; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
487; GFX1064-NEXT:    v_readlane_b32 s4, v1, 15
488; GFX1064-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
489; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
490; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
491; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
492; GFX1064-NEXT:    v_readlane_b32 s5, v1, 31
493; GFX1064-NEXT:    v_writelane_b32 v3, s4, 16
494; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
495; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
496; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
497; GFX1064-NEXT:    v_readlane_b32 s7, v1, 63
498; GFX1064-NEXT:    v_readlane_b32 s6, v1, 47
499; GFX1064-NEXT:    v_writelane_b32 v3, s5, 32
500; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
501; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
502; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
503; GFX1064-NEXT:    v_writelane_b32 v3, s6, 48
504; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
505; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
506; GFX1064-NEXT:    s_mov_b32 s2, -1
507; GFX1064-NEXT:    ; implicit-def: $vgpr0
508; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
509; GFX1064-NEXT:    s_cbranch_execz BB2_2
510; GFX1064-NEXT:  ; %bb.1:
511; GFX1064-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
512; GFX1064-NEXT:    v_mov_b32_e32 v4, s7
513; GFX1064-NEXT:    s_mov_b32 s3, s7
514; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
515; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
516; GFX1064-NEXT:    ds_add_rtn_u32 v0, v7, v4
517; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
518; GFX1064-NEXT:    buffer_gl0_inv
519; GFX1064-NEXT:    buffer_gl1_inv
520; GFX1064-NEXT:  BB2_2:
521; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
522; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
523; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
524; GFX1064-NEXT:    v_mov_b32_e32 v0, v3
525; GFX1064-NEXT:    v_add_nc_u32_e32 v0, s3, v0
526; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
527; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
528; GFX1064-NEXT:    s_nop 0
529; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
530; GFX1064-NEXT:    s_endpgm
531;
532; GFX1032-LABEL: add_i32_varying:
533; GFX1032:       ; %bb.0: ; %entry
534; GFX1032-NEXT:    v_mov_b32_e32 v1, v0
535; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
536; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
537; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
538; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
539; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
540; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
541; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
542; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
543; GFX1032-NEXT:    v_mov_b32_e32 v2, v1
544; GFX1032-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
545; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
546; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
547; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
548; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
549; GFX1032-NEXT:    v_mov_b32_e32 v3, 0
550; GFX1032-NEXT:    v_readlane_b32 s3, v1, 15
551; GFX1032-NEXT:    v_readlane_b32 s4, v1, 31
552; GFX1032-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
553; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
554; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
555; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
556; GFX1032-NEXT:    v_writelane_b32 v3, s3, 16
557; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
558; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
559; GFX1032-NEXT:    s_mov_b32 s2, -1
560; GFX1032-NEXT:    ; implicit-def: $vgpr0
561; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
562; GFX1032-NEXT:    s_cbranch_execz BB2_2
563; GFX1032-NEXT:  ; %bb.1:
564; GFX1032-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
565; GFX1032-NEXT:    v_mov_b32_e32 v4, s4
566; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
567; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
568; GFX1032-NEXT:    ds_add_rtn_u32 v0, v7, v4
569; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
570; GFX1032-NEXT:    buffer_gl0_inv
571; GFX1032-NEXT:    buffer_gl1_inv
572; GFX1032-NEXT:  BB2_2:
573; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
574; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s3
575; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
576; GFX1032-NEXT:    v_mov_b32_e32 v0, v3
577; GFX1032-NEXT:    v_add_nc_u32_e32 v0, s3, v0
578; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
579; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
580; GFX1032-NEXT:    s_nop 0
581; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
582; GFX1032-NEXT:    s_endpgm
583entry:
584  %lane = call i32 @llvm.amdgcn.workitem.id.x()
585  %old = atomicrmw add i32 addrspace(3)* @local_var32, i32 %lane acq_rel
586  store i32 %old, i32 addrspace(1)* %out
587  ret void
588}
589
590define amdgpu_kernel void @add_i32_varying_gfx1032(i32 addrspace(1)* %out) {
591;
592;
593; GFX7LESS-LABEL: add_i32_varying_gfx1032:
594; GFX7LESS:       ; %bb.0: ; %entry
595; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
596; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
597; GFX7LESS-NEXT:    s_mov_b32 m0, -1
598; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
599; GFX7LESS-NEXT:    ds_add_rtn_u32 v0, v1, v0
600; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
601; GFX7LESS-NEXT:    buffer_wbinvl1
602; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
603; GFX7LESS-NEXT:    s_mov_b32 s2, -1
604; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
605; GFX7LESS-NEXT:    s_endpgm
606;
607; GFX8-LABEL: add_i32_varying_gfx1032:
608; GFX8:       ; %bb.0: ; %entry
609; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
610; GFX8-NEXT:    v_mov_b32_e32 v2, v0
611; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
612; GFX8-NEXT:    v_mov_b32_e32 v1, 0
613; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
614; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
615; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
616; GFX8-NEXT:    s_not_b64 exec, exec
617; GFX8-NEXT:    v_mov_b32_e32 v2, 0
618; GFX8-NEXT:    s_not_b64 exec, exec
619; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
620; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
621; GFX8-NEXT:    s_nop 1
622; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
623; GFX8-NEXT:    s_nop 1
624; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
625; GFX8-NEXT:    s_nop 1
626; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
627; GFX8-NEXT:    s_nop 1
628; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
629; GFX8-NEXT:    s_nop 1
630; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
631; GFX8-NEXT:    v_readlane_b32 s4, v2, 63
632; GFX8-NEXT:    s_nop 0
633; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
634; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
635; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
636; GFX8-NEXT:    ; implicit-def: $vgpr0
637; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
638; GFX8-NEXT:    s_cbranch_execz BB3_2
639; GFX8-NEXT:  ; %bb.1:
640; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
641; GFX8-NEXT:    v_mov_b32_e32 v3, s4
642; GFX8-NEXT:    s_mov_b32 m0, -1
643; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
644; GFX8-NEXT:    ds_add_rtn_u32 v0, v0, v3
645; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
646; GFX8-NEXT:    buffer_wbinvl1_vol
647; GFX8-NEXT:  BB3_2:
648; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
649; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
650; GFX8-NEXT:    v_mov_b32_e32 v0, v1
651; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
652; GFX8-NEXT:    s_mov_b32 s3, 0xf000
653; GFX8-NEXT:    s_mov_b32 s2, -1
654; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
655; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
656; GFX8-NEXT:    s_endpgm
657;
658; GFX9-LABEL: add_i32_varying_gfx1032:
659; GFX9:       ; %bb.0: ; %entry
660; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
661; GFX9-NEXT:    v_mov_b32_e32 v2, v0
662; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
663; GFX9-NEXT:    v_mov_b32_e32 v1, 0
664; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
665; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
666; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
667; GFX9-NEXT:    s_not_b64 exec, exec
668; GFX9-NEXT:    v_mov_b32_e32 v2, 0
669; GFX9-NEXT:    s_not_b64 exec, exec
670; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
671; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
672; GFX9-NEXT:    s_nop 1
673; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
674; GFX9-NEXT:    s_nop 1
675; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
676; GFX9-NEXT:    s_nop 1
677; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
678; GFX9-NEXT:    s_nop 1
679; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
680; GFX9-NEXT:    s_nop 1
681; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
682; GFX9-NEXT:    v_readlane_b32 s4, v2, 63
683; GFX9-NEXT:    s_nop 0
684; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
685; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
686; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
687; GFX9-NEXT:    ; implicit-def: $vgpr0
688; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
689; GFX9-NEXT:    s_cbranch_execz BB3_2
690; GFX9-NEXT:  ; %bb.1:
691; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
692; GFX9-NEXT:    v_mov_b32_e32 v3, s4
693; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
694; GFX9-NEXT:    ds_add_rtn_u32 v0, v0, v3
695; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
696; GFX9-NEXT:    buffer_wbinvl1_vol
697; GFX9-NEXT:  BB3_2:
698; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
699; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
700; GFX9-NEXT:    v_mov_b32_e32 v0, v1
701; GFX9-NEXT:    v_add_u32_e32 v0, s2, v0
702; GFX9-NEXT:    s_mov_b32 s3, 0xf000
703; GFX9-NEXT:    s_mov_b32 s2, -1
704; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
705; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
706; GFX9-NEXT:    s_endpgm
707;
708; GFX1064-LABEL: add_i32_varying_gfx1032:
709; GFX1064:       ; %bb.0: ; %entry
710; GFX1064-NEXT:    v_mov_b32_e32 v1, v0
711; GFX1064-NEXT:    s_not_b64 exec, exec
712; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
713; GFX1064-NEXT:    s_not_b64 exec, exec
714; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
715; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
716; GFX1064-NEXT:    v_mov_b32_e32 v3, 0
717; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
718; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
719; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
720; GFX1064-NEXT:    v_mov_b32_e32 v2, v1
721; GFX1064-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
722; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
723; GFX1064-NEXT:    v_readlane_b32 s4, v1, 31
724; GFX1064-NEXT:    v_mov_b32_e32 v2, s4
725; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
726; GFX1064-NEXT:    v_readlane_b32 s4, v1, 15
727; GFX1064-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
728; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
729; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
730; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
731; GFX1064-NEXT:    v_readlane_b32 s5, v1, 31
732; GFX1064-NEXT:    v_writelane_b32 v3, s4, 16
733; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
734; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
735; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
736; GFX1064-NEXT:    v_readlane_b32 s7, v1, 63
737; GFX1064-NEXT:    v_readlane_b32 s6, v1, 47
738; GFX1064-NEXT:    v_writelane_b32 v3, s5, 32
739; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
740; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
741; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
742; GFX1064-NEXT:    v_writelane_b32 v3, s6, 48
743; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
744; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
745; GFX1064-NEXT:    s_mov_b32 s2, -1
746; GFX1064-NEXT:    ; implicit-def: $vgpr0
747; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
748; GFX1064-NEXT:    s_cbranch_execz BB3_2
749; GFX1064-NEXT:  ; %bb.1:
750; GFX1064-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
751; GFX1064-NEXT:    v_mov_b32_e32 v4, s7
752; GFX1064-NEXT:    s_mov_b32 s3, s7
753; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
754; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
755; GFX1064-NEXT:    ds_add_rtn_u32 v0, v7, v4
756; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
757; GFX1064-NEXT:    buffer_gl0_inv
758; GFX1064-NEXT:    buffer_gl1_inv
759; GFX1064-NEXT:  BB3_2:
760; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
761; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
762; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
763; GFX1064-NEXT:    v_mov_b32_e32 v0, v3
764; GFX1064-NEXT:    v_add_nc_u32_e32 v0, s3, v0
765; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
766; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
767; GFX1064-NEXT:    s_nop 0
768; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
769; GFX1064-NEXT:    s_endpgm
770;
771; GFX1032-LABEL: add_i32_varying_gfx1032:
772; GFX1032:       ; %bb.0: ; %entry
773; GFX1032-NEXT:    v_mov_b32_e32 v1, v0
774; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
775; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
776; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
777; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
778; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
779; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
780; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
781; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
782; GFX1032-NEXT:    v_mov_b32_e32 v2, v1
783; GFX1032-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
784; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
785; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
786; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
787; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
788; GFX1032-NEXT:    v_mov_b32_e32 v3, 0
789; GFX1032-NEXT:    v_readlane_b32 s3, v1, 15
790; GFX1032-NEXT:    v_readlane_b32 s4, v1, 31
791; GFX1032-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
792; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
793; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
794; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
795; GFX1032-NEXT:    v_writelane_b32 v3, s3, 16
796; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
797; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
798; GFX1032-NEXT:    s_mov_b32 s2, -1
799; GFX1032-NEXT:    ; implicit-def: $vgpr0
800; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
801; GFX1032-NEXT:    s_cbranch_execz BB3_2
802; GFX1032-NEXT:  ; %bb.1:
803; GFX1032-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
804; GFX1032-NEXT:    v_mov_b32_e32 v4, s4
805; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
806; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
807; GFX1032-NEXT:    ds_add_rtn_u32 v0, v7, v4
808; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
809; GFX1032-NEXT:    buffer_gl0_inv
810; GFX1032-NEXT:    buffer_gl1_inv
811; GFX1032-NEXT:  BB3_2:
812; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
813; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s3
814; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
815; GFX1032-NEXT:    v_mov_b32_e32 v0, v3
816; GFX1032-NEXT:    v_add_nc_u32_e32 v0, s3, v0
817; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
818; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
819; GFX1032-NEXT:    s_nop 0
820; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
821; GFX1032-NEXT:    s_endpgm
822entry:
823  %lane = call i32 @llvm.amdgcn.workitem.id.x()
824  %old = atomicrmw add i32 addrspace(3)* @local_var32, i32 %lane acq_rel
825  store i32 %old, i32 addrspace(1)* %out
826  ret void
827}
828
829define amdgpu_kernel void @add_i32_varying_gfx1064(i32 addrspace(1)* %out) {
830;
831;
832; GFX7LESS-LABEL: add_i32_varying_gfx1064:
833; GFX7LESS:       ; %bb.0: ; %entry
834; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
835; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
836; GFX7LESS-NEXT:    s_mov_b32 m0, -1
837; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
838; GFX7LESS-NEXT:    ds_add_rtn_u32 v0, v1, v0
839; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
840; GFX7LESS-NEXT:    buffer_wbinvl1
841; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
842; GFX7LESS-NEXT:    s_mov_b32 s2, -1
843; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
844; GFX7LESS-NEXT:    s_endpgm
845;
846; GFX8-LABEL: add_i32_varying_gfx1064:
847; GFX8:       ; %bb.0: ; %entry
848; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
849; GFX8-NEXT:    v_mov_b32_e32 v2, v0
850; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
851; GFX8-NEXT:    v_mov_b32_e32 v1, 0
852; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
853; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
854; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
855; GFX8-NEXT:    s_not_b64 exec, exec
856; GFX8-NEXT:    v_mov_b32_e32 v2, 0
857; GFX8-NEXT:    s_not_b64 exec, exec
858; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
859; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
860; GFX8-NEXT:    s_nop 1
861; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
862; GFX8-NEXT:    s_nop 1
863; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
864; GFX8-NEXT:    s_nop 1
865; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
866; GFX8-NEXT:    s_nop 1
867; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
868; GFX8-NEXT:    s_nop 1
869; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
870; GFX8-NEXT:    v_readlane_b32 s4, v2, 63
871; GFX8-NEXT:    s_nop 0
872; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
873; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
874; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
875; GFX8-NEXT:    ; implicit-def: $vgpr0
876; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
877; GFX8-NEXT:    s_cbranch_execz BB4_2
878; GFX8-NEXT:  ; %bb.1:
879; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
880; GFX8-NEXT:    v_mov_b32_e32 v3, s4
881; GFX8-NEXT:    s_mov_b32 m0, -1
882; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
883; GFX8-NEXT:    ds_add_rtn_u32 v0, v0, v3
884; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
885; GFX8-NEXT:    buffer_wbinvl1_vol
886; GFX8-NEXT:  BB4_2:
887; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
888; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
889; GFX8-NEXT:    v_mov_b32_e32 v0, v1
890; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
891; GFX8-NEXT:    s_mov_b32 s3, 0xf000
892; GFX8-NEXT:    s_mov_b32 s2, -1
893; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
894; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
895; GFX8-NEXT:    s_endpgm
896;
897; GFX9-LABEL: add_i32_varying_gfx1064:
898; GFX9:       ; %bb.0: ; %entry
899; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
900; GFX9-NEXT:    v_mov_b32_e32 v2, v0
901; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
902; GFX9-NEXT:    v_mov_b32_e32 v1, 0
903; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
904; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
905; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
906; GFX9-NEXT:    s_not_b64 exec, exec
907; GFX9-NEXT:    v_mov_b32_e32 v2, 0
908; GFX9-NEXT:    s_not_b64 exec, exec
909; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
910; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
911; GFX9-NEXT:    s_nop 1
912; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
913; GFX9-NEXT:    s_nop 1
914; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
915; GFX9-NEXT:    s_nop 1
916; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
917; GFX9-NEXT:    s_nop 1
918; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
919; GFX9-NEXT:    s_nop 1
920; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
921; GFX9-NEXT:    v_readlane_b32 s4, v2, 63
922; GFX9-NEXT:    s_nop 0
923; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
924; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
925; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
926; GFX9-NEXT:    ; implicit-def: $vgpr0
927; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
928; GFX9-NEXT:    s_cbranch_execz BB4_2
929; GFX9-NEXT:  ; %bb.1:
930; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
931; GFX9-NEXT:    v_mov_b32_e32 v3, s4
932; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
933; GFX9-NEXT:    ds_add_rtn_u32 v0, v0, v3
934; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
935; GFX9-NEXT:    buffer_wbinvl1_vol
936; GFX9-NEXT:  BB4_2:
937; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
938; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
939; GFX9-NEXT:    v_mov_b32_e32 v0, v1
940; GFX9-NEXT:    v_add_u32_e32 v0, s2, v0
941; GFX9-NEXT:    s_mov_b32 s3, 0xf000
942; GFX9-NEXT:    s_mov_b32 s2, -1
943; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
944; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
945; GFX9-NEXT:    s_endpgm
946;
947; GFX1064-LABEL: add_i32_varying_gfx1064:
948; GFX1064:       ; %bb.0: ; %entry
949; GFX1064-NEXT:    v_mov_b32_e32 v1, v0
950; GFX1064-NEXT:    s_not_b64 exec, exec
951; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
952; GFX1064-NEXT:    s_not_b64 exec, exec
953; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
954; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
955; GFX1064-NEXT:    v_mov_b32_e32 v3, 0
956; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
957; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
958; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
959; GFX1064-NEXT:    v_mov_b32_e32 v2, v1
960; GFX1064-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
961; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
962; GFX1064-NEXT:    v_readlane_b32 s4, v1, 31
963; GFX1064-NEXT:    v_mov_b32_e32 v2, s4
964; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
965; GFX1064-NEXT:    v_readlane_b32 s4, v1, 15
966; GFX1064-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
967; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
968; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
969; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
970; GFX1064-NEXT:    v_readlane_b32 s5, v1, 31
971; GFX1064-NEXT:    v_writelane_b32 v3, s4, 16
972; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
973; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
974; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
975; GFX1064-NEXT:    v_readlane_b32 s7, v1, 63
976; GFX1064-NEXT:    v_readlane_b32 s6, v1, 47
977; GFX1064-NEXT:    v_writelane_b32 v3, s5, 32
978; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
979; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
980; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
981; GFX1064-NEXT:    v_writelane_b32 v3, s6, 48
982; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
983; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
984; GFX1064-NEXT:    s_mov_b32 s2, -1
985; GFX1064-NEXT:    ; implicit-def: $vgpr0
986; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
987; GFX1064-NEXT:    s_cbranch_execz BB4_2
988; GFX1064-NEXT:  ; %bb.1:
989; GFX1064-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
990; GFX1064-NEXT:    v_mov_b32_e32 v4, s7
991; GFX1064-NEXT:    s_mov_b32 s3, s7
992; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
993; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
994; GFX1064-NEXT:    ds_add_rtn_u32 v0, v7, v4
995; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
996; GFX1064-NEXT:    buffer_gl0_inv
997; GFX1064-NEXT:    buffer_gl1_inv
998; GFX1064-NEXT:  BB4_2:
999; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
1000; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
1001; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
1002; GFX1064-NEXT:    v_mov_b32_e32 v0, v3
1003; GFX1064-NEXT:    v_add_nc_u32_e32 v0, s3, v0
1004; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
1005; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
1006; GFX1064-NEXT:    s_nop 0
1007; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1008; GFX1064-NEXT:    s_endpgm
1009;
1010; GFX1032-LABEL: add_i32_varying_gfx1064:
1011; GFX1032:       ; %bb.0: ; %entry
1012; GFX1032-NEXT:    v_mov_b32_e32 v1, v0
1013; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
1014; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
1015; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
1016; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
1017; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
1018; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
1019; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
1020; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
1021; GFX1032-NEXT:    v_mov_b32_e32 v2, v1
1022; GFX1032-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
1023; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
1024; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1025; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
1026; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
1027; GFX1032-NEXT:    v_mov_b32_e32 v3, 0
1028; GFX1032-NEXT:    v_readlane_b32 s3, v1, 15
1029; GFX1032-NEXT:    v_readlane_b32 s4, v1, 31
1030; GFX1032-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
1031; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
1032; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
1033; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
1034; GFX1032-NEXT:    v_writelane_b32 v3, s3, 16
1035; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
1036; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
1037; GFX1032-NEXT:    s_mov_b32 s2, -1
1038; GFX1032-NEXT:    ; implicit-def: $vgpr0
1039; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
1040; GFX1032-NEXT:    s_cbranch_execz BB4_2
1041; GFX1032-NEXT:  ; %bb.1:
1042; GFX1032-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
1043; GFX1032-NEXT:    v_mov_b32_e32 v4, s4
1044; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1045; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
1046; GFX1032-NEXT:    ds_add_rtn_u32 v0, v7, v4
1047; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1048; GFX1032-NEXT:    buffer_gl0_inv
1049; GFX1032-NEXT:    buffer_gl1_inv
1050; GFX1032-NEXT:  BB4_2:
1051; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
1052; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s3
1053; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
1054; GFX1032-NEXT:    v_mov_b32_e32 v0, v3
1055; GFX1032-NEXT:    v_add_nc_u32_e32 v0, s3, v0
1056; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
1057; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
1058; GFX1032-NEXT:    s_nop 0
1059; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1060; GFX1032-NEXT:    s_endpgm
1061entry:
1062  %lane = call i32 @llvm.amdgcn.workitem.id.x()
1063  %old = atomicrmw add i32 addrspace(3)* @local_var32, i32 %lane acq_rel
1064  store i32 %old, i32 addrspace(1)* %out
1065  ret void
1066}
1067
1068define amdgpu_kernel void @add_i64_constant(i64 addrspace(1)* %out) {
1069;
1070;
1071; GFX7LESS-LABEL: add_i64_constant:
1072; GFX7LESS:       ; %bb.0: ; %entry
1073; GFX7LESS-NEXT:    s_mov_b64 s[4:5], exec
1074; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
1075; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
1076; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s5, v0
1077; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
1078; GFX7LESS-NEXT:    ; implicit-def: $vgpr1_vgpr2
1079; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
1080; GFX7LESS-NEXT:    s_cbranch_execz BB5_2
1081; GFX7LESS-NEXT:  ; %bb.1:
1082; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
1083; GFX7LESS-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
1084; GFX7LESS-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
1085; GFX7LESS-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
1086; GFX7LESS-NEXT:    s_mov_b32 m0, -1
1087; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1088; GFX7LESS-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
1089; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1090; GFX7LESS-NEXT:    buffer_wbinvl1
1091; GFX7LESS-NEXT:  BB5_2:
1092; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
1093; GFX7LESS-NEXT:    v_readfirstlane_b32 s2, v1
1094; GFX7LESS-NEXT:    v_readfirstlane_b32 s4, v2
1095; GFX7LESS-NEXT:    v_mul_hi_u32_u24_e32 v1, 5, v0
1096; GFX7LESS-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
1097; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
1098; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s4
1099; GFX7LESS-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
1100; GFX7LESS-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
1101; GFX7LESS-NEXT:    s_mov_b32 s2, -1
1102; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
1103; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
1104; GFX7LESS-NEXT:    s_endpgm
1105;
1106; GFX8-LABEL: add_i64_constant:
1107; GFX8:       ; %bb.0: ; %entry
1108; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1109; GFX8-NEXT:    s_mov_b64 s[4:5], exec
1110; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
1111; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
1112; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
1113; GFX8-NEXT:    ; implicit-def: $vgpr1_vgpr2
1114; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
1115; GFX8-NEXT:    s_cbranch_execz BB5_2
1116; GFX8-NEXT:  ; %bb.1:
1117; GFX8-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
1118; GFX8-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
1119; GFX8-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
1120; GFX8-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
1121; GFX8-NEXT:    s_mov_b32 m0, -1
1122; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1123; GFX8-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
1124; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1125; GFX8-NEXT:    buffer_wbinvl1_vol
1126; GFX8-NEXT:  BB5_2:
1127; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
1128; GFX8-NEXT:    v_readfirstlane_b32 s2, v1
1129; GFX8-NEXT:    v_readfirstlane_b32 s3, v2
1130; GFX8-NEXT:    v_mov_b32_e32 v1, s2
1131; GFX8-NEXT:    v_mov_b32_e32 v2, s3
1132; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[2:3], v0, 5, v[1:2]
1133; GFX8-NEXT:    s_mov_b32 s3, 0xf000
1134; GFX8-NEXT:    s_mov_b32 s2, -1
1135; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1136; GFX8-NEXT:    s_nop 1
1137; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
1138; GFX8-NEXT:    s_endpgm
1139;
1140; GFX9-LABEL: add_i64_constant:
1141; GFX9:       ; %bb.0: ; %entry
1142; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1143; GFX9-NEXT:    s_mov_b64 s[4:5], exec
1144; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
1145; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
1146; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
1147; GFX9-NEXT:    ; implicit-def: $vgpr1_vgpr2
1148; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
1149; GFX9-NEXT:    s_cbranch_execz BB5_2
1150; GFX9-NEXT:  ; %bb.1:
1151; GFX9-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
1152; GFX9-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
1153; GFX9-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
1154; GFX9-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
1155; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1156; GFX9-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
1157; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1158; GFX9-NEXT:    buffer_wbinvl1_vol
1159; GFX9-NEXT:  BB5_2:
1160; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
1161; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
1162; GFX9-NEXT:    v_readfirstlane_b32 s3, v2
1163; GFX9-NEXT:    v_mov_b32_e32 v1, s2
1164; GFX9-NEXT:    v_mov_b32_e32 v2, s3
1165; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[2:3], v0, 5, v[1:2]
1166; GFX9-NEXT:    s_mov_b32 s3, 0xf000
1167; GFX9-NEXT:    s_mov_b32 s2, -1
1168; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1169; GFX9-NEXT:    s_nop 1
1170; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
1171; GFX9-NEXT:    s_endpgm
1172;
1173; GFX1064-LABEL: add_i64_constant:
1174; GFX1064:       ; %bb.0: ; %entry
1175; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1176; GFX1064-NEXT:    s_mov_b64 s[4:5], exec
1177; GFX1064-NEXT:    ; implicit-def: $vgpr1_vgpr2
1178; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
1179; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s5, v0
1180; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
1181; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
1182; GFX1064-NEXT:    s_cbranch_execz BB5_2
1183; GFX1064-NEXT:  ; %bb.1:
1184; GFX1064-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
1185; GFX1064-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
1186; GFX1064-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
1187; GFX1064-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
1188; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1189; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
1190; GFX1064-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
1191; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1192; GFX1064-NEXT:    buffer_gl0_inv
1193; GFX1064-NEXT:    buffer_gl1_inv
1194; GFX1064-NEXT:  BB5_2:
1195; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
1196; GFX1064-NEXT:    s_or_b64 exec, exec, s[2:3]
1197; GFX1064-NEXT:    v_readfirstlane_b32 s2, v1
1198; GFX1064-NEXT:    v_readfirstlane_b32 s3, v2
1199; GFX1064-NEXT:    v_mad_u64_u32 v[0:1], s[2:3], v0, 5, s[2:3]
1200; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
1201; GFX1064-NEXT:    s_mov_b32 s2, -1
1202; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
1203; GFX1064-NEXT:    s_nop 1
1204; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
1205; GFX1064-NEXT:    s_endpgm
1206;
1207; GFX1032-LABEL: add_i64_constant:
1208; GFX1032:       ; %bb.0: ; %entry
1209; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1210; GFX1032-NEXT:    s_mov_b32 s3, exec_lo
1211; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
1212; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
1213; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
1214; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
1215; GFX1032-NEXT:    s_cbranch_execz BB5_2
1216; GFX1032-NEXT:  ; %bb.1:
1217; GFX1032-NEXT:    s_bcnt1_i32_b32 s3, s3
1218; GFX1032-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
1219; GFX1032-NEXT:    v_mul_hi_u32_u24_e64 v2, s3, 5
1220; GFX1032-NEXT:    v_mul_u32_u24_e64 v1, s3, 5
1221; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1222; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
1223; GFX1032-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
1224; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1225; GFX1032-NEXT:    buffer_gl0_inv
1226; GFX1032-NEXT:    buffer_gl1_inv
1227; GFX1032-NEXT:  BB5_2:
1228; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
1229; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2
1230; GFX1032-NEXT:    v_readfirstlane_b32 s2, v1
1231; GFX1032-NEXT:    v_readfirstlane_b32 s3, v2
1232; GFX1032-NEXT:    v_mad_u64_u32 v[0:1], s2, v0, 5, s[2:3]
1233; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
1234; GFX1032-NEXT:    s_mov_b32 s2, -1
1235; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
1236; GFX1032-NEXT:    s_nop 1
1237; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
1238; GFX1032-NEXT:    s_endpgm
1239entry:
1240  %old = atomicrmw add i64 addrspace(3)* @local_var64, i64 5 acq_rel
1241  store i64 %old, i64 addrspace(1)* %out
1242  ret void
1243}
1244
1245define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 %additive) {
1246;
1247;
1248; GFX7LESS-LABEL: add_i64_uniform:
1249; GFX7LESS:       ; %bb.0: ; %entry
1250; GFX7LESS-NEXT:    s_mov_b64 s[6:7], exec
1251; GFX7LESS-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
1252; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
1253; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s7, v0
1254; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
1255; GFX7LESS-NEXT:    ; implicit-def: $vgpr1_vgpr2
1256; GFX7LESS-NEXT:    s_and_saveexec_b64 s[4:5], vcc
1257; GFX7LESS-NEXT:    s_cbranch_execz BB6_2
1258; GFX7LESS-NEXT:  ; %bb.1:
1259; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
1260; GFX7LESS-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
1261; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
1262; GFX7LESS-NEXT:    s_mul_i32 s7, s3, s6
1263; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s6
1264; GFX7LESS-NEXT:    v_mul_hi_u32 v1, s2, v1
1265; GFX7LESS-NEXT:    s_mul_i32 s6, s2, s6
1266; GFX7LESS-NEXT:    v_add_i32_e32 v2, vcc, s7, v1
1267; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s6
1268; GFX7LESS-NEXT:    s_mov_b32 m0, -1
1269; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1270; GFX7LESS-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
1271; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1272; GFX7LESS-NEXT:    buffer_wbinvl1
1273; GFX7LESS-NEXT:  BB6_2:
1274; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[4:5]
1275; GFX7LESS-NEXT:    s_mov_b32 s7, 0xf000
1276; GFX7LESS-NEXT:    s_mov_b32 s6, -1
1277; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
1278; GFX7LESS-NEXT:    s_mov_b32 s4, s0
1279; GFX7LESS-NEXT:    s_mov_b32 s5, s1
1280; GFX7LESS-NEXT:    v_readfirstlane_b32 s0, v1
1281; GFX7LESS-NEXT:    v_readfirstlane_b32 s1, v2
1282; GFX7LESS-NEXT:    v_mul_lo_u32 v1, s3, v0
1283; GFX7LESS-NEXT:    v_mul_hi_u32 v2, s2, v0
1284; GFX7LESS-NEXT:    v_mul_lo_u32 v0, s2, v0
1285; GFX7LESS-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
1286; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s1
1287; GFX7LESS-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
1288; GFX7LESS-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
1289; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
1290; GFX7LESS-NEXT:    s_endpgm
1291;
1292; GFX8-LABEL: add_i64_uniform:
1293; GFX8:       ; %bb.0: ; %entry
1294; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
1295; GFX8-NEXT:    s_mov_b64 s[6:7], exec
1296; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
1297; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
1298; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
1299; GFX8-NEXT:    ; implicit-def: $vgpr1_vgpr2
1300; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
1301; GFX8-NEXT:    s_cbranch_execz BB6_2
1302; GFX8-NEXT:  ; %bb.1:
1303; GFX8-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
1304; GFX8-NEXT:    v_mov_b32_e32 v1, s6
1305; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1306; GFX8-NEXT:    v_mul_hi_u32 v1, s2, v1
1307; GFX8-NEXT:    s_mul_i32 s7, s3, s6
1308; GFX8-NEXT:    s_mul_i32 s6, s2, s6
1309; GFX8-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
1310; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s7, v1
1311; GFX8-NEXT:    v_mov_b32_e32 v1, s6
1312; GFX8-NEXT:    s_mov_b32 m0, -1
1313; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1314; GFX8-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
1315; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1316; GFX8-NEXT:    buffer_wbinvl1_vol
1317; GFX8-NEXT:  BB6_2:
1318; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
1319; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1320; GFX8-NEXT:    s_mov_b32 s4, s0
1321; GFX8-NEXT:    v_readfirstlane_b32 s0, v1
1322; GFX8-NEXT:    v_mul_lo_u32 v1, s3, v0
1323; GFX8-NEXT:    v_mul_hi_u32 v3, s2, v0
1324; GFX8-NEXT:    v_mul_lo_u32 v0, s2, v0
1325; GFX8-NEXT:    s_mov_b32 s5, s1
1326; GFX8-NEXT:    v_readfirstlane_b32 s1, v2
1327; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v3, v1
1328; GFX8-NEXT:    v_mov_b32_e32 v2, s1
1329; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
1330; GFX8-NEXT:    s_mov_b32 s7, 0xf000
1331; GFX8-NEXT:    s_mov_b32 s6, -1
1332; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
1333; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
1334; GFX8-NEXT:    s_endpgm
1335;
1336; GFX9-LABEL: add_i64_uniform:
1337; GFX9:       ; %bb.0: ; %entry
1338; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
1339; GFX9-NEXT:    s_mov_b64 s[6:7], exec
1340; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
1341; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
1342; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
1343; GFX9-NEXT:    ; implicit-def: $vgpr1_vgpr2
1344; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
1345; GFX9-NEXT:    s_cbranch_execz BB6_2
1346; GFX9-NEXT:  ; %bb.1:
1347; GFX9-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
1348; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1349; GFX9-NEXT:    s_mul_i32 s7, s3, s6
1350; GFX9-NEXT:    s_mul_hi_u32 s8, s2, s6
1351; GFX9-NEXT:    s_add_i32 s8, s8, s7
1352; GFX9-NEXT:    s_mul_i32 s6, s2, s6
1353; GFX9-NEXT:    v_mov_b32_e32 v1, s6
1354; GFX9-NEXT:    v_mov_b32_e32 v2, s8
1355; GFX9-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
1356; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1357; GFX9-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
1358; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1359; GFX9-NEXT:    buffer_wbinvl1_vol
1360; GFX9-NEXT:  BB6_2:
1361; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
1362; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1363; GFX9-NEXT:    v_mul_lo_u32 v3, s3, v0
1364; GFX9-NEXT:    v_mul_hi_u32 v4, s2, v0
1365; GFX9-NEXT:    v_mul_lo_u32 v0, s2, v0
1366; GFX9-NEXT:    s_mov_b32 s4, s0
1367; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
1368; GFX9-NEXT:    s_mov_b32 s5, s1
1369; GFX9-NEXT:    v_readfirstlane_b32 s1, v2
1370; GFX9-NEXT:    v_add_u32_e32 v1, v4, v3
1371; GFX9-NEXT:    v_mov_b32_e32 v2, s1
1372; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
1373; GFX9-NEXT:    s_mov_b32 s7, 0xf000
1374; GFX9-NEXT:    s_mov_b32 s6, -1
1375; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
1376; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
1377; GFX9-NEXT:    s_endpgm
1378;
1379; GFX1064-LABEL: add_i64_uniform:
1380; GFX1064:       ; %bb.0: ; %entry
1381; GFX1064-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
1382; GFX1064-NEXT:    s_mov_b64 s[6:7], exec
1383; GFX1064-NEXT:    ; implicit-def: $vgpr1_vgpr2
1384; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
1385; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s7, v0
1386; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
1387; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
1388; GFX1064-NEXT:    s_cbranch_execz BB6_2
1389; GFX1064-NEXT:  ; %bb.1:
1390; GFX1064-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
1391; GFX1064-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
1392; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
1393; GFX1064-NEXT:    s_mul_i32 s7, s3, s6
1394; GFX1064-NEXT:    s_mul_hi_u32 s8, s2, s6
1395; GFX1064-NEXT:    s_mul_i32 s6, s2, s6
1396; GFX1064-NEXT:    s_add_i32 s8, s8, s7
1397; GFX1064-NEXT:    v_mov_b32_e32 v1, s6
1398; GFX1064-NEXT:    v_mov_b32_e32 v2, s8
1399; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1400; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
1401; GFX1064-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
1402; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1403; GFX1064-NEXT:    buffer_gl0_inv
1404; GFX1064-NEXT:    buffer_gl1_inv
1405; GFX1064-NEXT:  BB6_2:
1406; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
1407; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
1408; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
1409; GFX1064-NEXT:    v_mul_lo_u32 v3, s3, v0
1410; GFX1064-NEXT:    v_mul_hi_u32 v4, s2, v0
1411; GFX1064-NEXT:    v_mul_lo_u32 v0, s2, v0
1412; GFX1064-NEXT:    v_readfirstlane_b32 s2, v1
1413; GFX1064-NEXT:    v_readfirstlane_b32 s4, v2
1414; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
1415; GFX1064-NEXT:    v_add_nc_u32_e32 v1, v4, v3
1416; GFX1064-NEXT:    v_add_co_u32_e64 v0, vcc, s2, v0
1417; GFX1064-NEXT:    s_mov_b32 s2, -1
1418; GFX1064-NEXT:    v_add_co_ci_u32_e32 v1, vcc, s4, v1, vcc
1419; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
1420; GFX1064-NEXT:    s_endpgm
1421;
1422; GFX1032-LABEL: add_i64_uniform:
1423; GFX1032:       ; %bb.0: ; %entry
1424; GFX1032-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
1425; GFX1032-NEXT:    s_mov_b32 s5, exec_lo
1426; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
1427; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s5, 0
1428; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
1429; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
1430; GFX1032-NEXT:    s_cbranch_execz BB6_2
1431; GFX1032-NEXT:  ; %bb.1:
1432; GFX1032-NEXT:    s_bcnt1_i32_b32 s5, s5
1433; GFX1032-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
1434; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
1435; GFX1032-NEXT:    s_mul_i32 s6, s3, s5
1436; GFX1032-NEXT:    s_mul_hi_u32 s7, s2, s5
1437; GFX1032-NEXT:    s_mul_i32 s5, s2, s5
1438; GFX1032-NEXT:    s_add_i32 s7, s7, s6
1439; GFX1032-NEXT:    v_mov_b32_e32 v1, s5
1440; GFX1032-NEXT:    v_mov_b32_e32 v2, s7
1441; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1442; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
1443; GFX1032-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
1444; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1445; GFX1032-NEXT:    buffer_gl0_inv
1446; GFX1032-NEXT:    buffer_gl1_inv
1447; GFX1032-NEXT:  BB6_2:
1448; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
1449; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
1450; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
1451; GFX1032-NEXT:    v_mul_lo_u32 v3, s3, v0
1452; GFX1032-NEXT:    v_mul_hi_u32 v4, s2, v0
1453; GFX1032-NEXT:    v_mul_lo_u32 v0, s2, v0
1454; GFX1032-NEXT:    v_readfirstlane_b32 s2, v1
1455; GFX1032-NEXT:    v_readfirstlane_b32 s4, v2
1456; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
1457; GFX1032-NEXT:    v_add_nc_u32_e32 v1, v4, v3
1458; GFX1032-NEXT:    v_add_co_u32_e64 v0, vcc_lo, s2, v0
1459; GFX1032-NEXT:    s_mov_b32 s2, -1
1460; GFX1032-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, s4, v1, vcc_lo
1461; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
1462; GFX1032-NEXT:    s_endpgm
1463entry:
1464  %old = atomicrmw add i64 addrspace(3)* @local_var64, i64 %additive acq_rel
1465  store i64 %old, i64 addrspace(1)* %out
1466  ret void
1467}
1468
1469; GCN-NOT: v_mbcnt_lo_u32_b32
1470; GCN-NOT: v_mbcnt_hi_u32_b32
1471; GCN-NOT: s_bcnt1_i32_b64
1472define amdgpu_kernel void @add_i64_varying(i64 addrspace(1)* %out) {
1473;
1474;
1475; GFX7LESS-LABEL: add_i64_varying:
1476; GFX7LESS:       ; %bb.0: ; %entry
1477; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
1478; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
1479; GFX7LESS-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
1480; GFX7LESS-NEXT:    s_mov_b32 m0, -1
1481; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1482; GFX7LESS-NEXT:    ds_add_rtn_u64 v[0:1], v2, v[0:1]
1483; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1484; GFX7LESS-NEXT:    buffer_wbinvl1
1485; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
1486; GFX7LESS-NEXT:    s_mov_b32 s2, -1
1487; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
1488; GFX7LESS-NEXT:    s_endpgm
1489;
1490; GFX8-LABEL: add_i64_varying:
1491; GFX8:       ; %bb.0: ; %entry
1492; GFX8-NEXT:    v_mov_b32_e32 v1, 0
1493; GFX8-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
1494; GFX8-NEXT:    s_mov_b32 m0, -1
1495; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1496; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1497; GFX8-NEXT:    ds_add_rtn_u64 v[0:1], v2, v[0:1]
1498; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1499; GFX8-NEXT:    buffer_wbinvl1_vol
1500; GFX8-NEXT:    s_mov_b32 s3, 0xf000
1501; GFX8-NEXT:    s_mov_b32 s2, -1
1502; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
1503; GFX8-NEXT:    s_endpgm
1504;
1505; GFX9-LABEL: add_i64_varying:
1506; GFX9:       ; %bb.0: ; %entry
1507; GFX9-NEXT:    v_mov_b32_e32 v1, 0
1508; GFX9-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
1509; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1510; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1511; GFX9-NEXT:    ds_add_rtn_u64 v[0:1], v2, v[0:1]
1512; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1513; GFX9-NEXT:    buffer_wbinvl1_vol
1514; GFX9-NEXT:    s_mov_b32 s3, 0xf000
1515; GFX9-NEXT:    s_mov_b32 s2, -1
1516; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
1517; GFX9-NEXT:    s_endpgm
1518;
1519; GFX1064-LABEL: add_i64_varying:
1520; GFX1064:       ; %bb.0: ; %entry
1521; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
1522; GFX1064-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
1523; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1524; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
1525; GFX1064-NEXT:    s_mov_b32 s2, -1
1526; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1527; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
1528; GFX1064-NEXT:    ds_add_rtn_u64 v[0:1], v2, v[0:1]
1529; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1530; GFX1064-NEXT:    buffer_gl0_inv
1531; GFX1064-NEXT:    buffer_gl1_inv
1532; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
1533; GFX1064-NEXT:    s_endpgm
1534;
1535; GFX1032-LABEL: add_i64_varying:
1536; GFX1032:       ; %bb.0: ; %entry
1537; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
1538; GFX1032-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
1539; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1540; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
1541; GFX1032-NEXT:    s_mov_b32 s2, -1
1542; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1543; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
1544; GFX1032-NEXT:    ds_add_rtn_u64 v[0:1], v2, v[0:1]
1545; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1546; GFX1032-NEXT:    buffer_gl0_inv
1547; GFX1032-NEXT:    buffer_gl1_inv
1548; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
1549; GFX1032-NEXT:    s_endpgm
1550entry:
1551  %lane = call i32 @llvm.amdgcn.workitem.id.x()
1552  %zext = zext i32 %lane to i64
1553  %old = atomicrmw add i64 addrspace(3)* @local_var64, i64 %zext acq_rel
1554  store i64 %old, i64 addrspace(1)* %out
1555  ret void
1556}
1557
1558define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
1559;
1560;
1561; GFX7LESS-LABEL: sub_i32_constant:
1562; GFX7LESS:       ; %bb.0: ; %entry
1563; GFX7LESS-NEXT:    s_mov_b64 s[2:3], exec
1564; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
1565; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
1566; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
1567; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
1568; GFX7LESS-NEXT:    ; implicit-def: $vgpr1
1569; GFX7LESS-NEXT:    s_and_saveexec_b64 s[4:5], vcc
1570; GFX7LESS-NEXT:    s_cbranch_execz BB8_2
1571; GFX7LESS-NEXT:  ; %bb.1:
1572; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
1573; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
1574; GFX7LESS-NEXT:    v_mul_u32_u24_e64 v2, s2, 5
1575; GFX7LESS-NEXT:    s_mov_b32 m0, -1
1576; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1577; GFX7LESS-NEXT:    ds_sub_rtn_u32 v1, v1, v2
1578; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1579; GFX7LESS-NEXT:    buffer_wbinvl1
1580; GFX7LESS-NEXT:  BB8_2:
1581; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[4:5]
1582; GFX7LESS-NEXT:    v_readfirstlane_b32 s2, v1
1583; GFX7LESS-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
1584; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
1585; GFX7LESS-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
1586; GFX7LESS-NEXT:    s_mov_b32 s2, -1
1587; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
1588; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1589; GFX7LESS-NEXT:    s_endpgm
1590;
1591; GFX8-LABEL: sub_i32_constant:
1592; GFX8:       ; %bb.0: ; %entry
1593; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1594; GFX8-NEXT:    s_mov_b64 s[2:3], exec
1595; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
1596; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
1597; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
1598; GFX8-NEXT:    ; implicit-def: $vgpr1
1599; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
1600; GFX8-NEXT:    s_cbranch_execz BB8_2
1601; GFX8-NEXT:  ; %bb.1:
1602; GFX8-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
1603; GFX8-NEXT:    v_mul_u32_u24_e64 v1, s2, 5
1604; GFX8-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
1605; GFX8-NEXT:    s_mov_b32 m0, -1
1606; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1607; GFX8-NEXT:    ds_sub_rtn_u32 v1, v2, v1
1608; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1609; GFX8-NEXT:    buffer_wbinvl1_vol
1610; GFX8-NEXT:  BB8_2:
1611; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
1612; GFX8-NEXT:    v_readfirstlane_b32 s2, v1
1613; GFX8-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
1614; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s2, v0
1615; GFX8-NEXT:    s_mov_b32 s3, 0xf000
1616; GFX8-NEXT:    s_mov_b32 s2, -1
1617; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1618; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1619; GFX8-NEXT:    s_endpgm
1620;
1621; GFX9-LABEL: sub_i32_constant:
1622; GFX9:       ; %bb.0: ; %entry
1623; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1624; GFX9-NEXT:    s_mov_b64 s[2:3], exec
1625; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
1626; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
1627; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
1628; GFX9-NEXT:    ; implicit-def: $vgpr1
1629; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
1630; GFX9-NEXT:    s_cbranch_execz BB8_2
1631; GFX9-NEXT:  ; %bb.1:
1632; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
1633; GFX9-NEXT:    v_mul_u32_u24_e64 v1, s2, 5
1634; GFX9-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
1635; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1636; GFX9-NEXT:    ds_sub_rtn_u32 v1, v2, v1
1637; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1638; GFX9-NEXT:    buffer_wbinvl1_vol
1639; GFX9-NEXT:  BB8_2:
1640; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
1641; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
1642; GFX9-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
1643; GFX9-NEXT:    v_sub_u32_e32 v0, s2, v0
1644; GFX9-NEXT:    s_mov_b32 s3, 0xf000
1645; GFX9-NEXT:    s_mov_b32 s2, -1
1646; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1647; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1648; GFX9-NEXT:    s_endpgm
1649;
1650; GFX1064-LABEL: sub_i32_constant:
1651; GFX1064:       ; %bb.0: ; %entry
1652; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1653; GFX1064-NEXT:    s_mov_b64 s[2:3], exec
1654; GFX1064-NEXT:    ; implicit-def: $vgpr1
1655; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
1656; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
1657; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
1658; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
1659; GFX1064-NEXT:    s_cbranch_execz BB8_2
1660; GFX1064-NEXT:  ; %bb.1:
1661; GFX1064-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
1662; GFX1064-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
1663; GFX1064-NEXT:    v_mul_u32_u24_e64 v1, s2, 5
1664; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1665; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
1666; GFX1064-NEXT:    ds_sub_rtn_u32 v1, v2, v1
1667; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1668; GFX1064-NEXT:    buffer_gl0_inv
1669; GFX1064-NEXT:    buffer_gl1_inv
1670; GFX1064-NEXT:  BB8_2:
1671; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
1672; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
1673; GFX1064-NEXT:    v_readfirstlane_b32 s2, v1
1674; GFX1064-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
1675; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
1676; GFX1064-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
1677; GFX1064-NEXT:    s_mov_b32 s2, -1
1678; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
1679; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1680; GFX1064-NEXT:    s_endpgm
1681;
1682; GFX1032-LABEL: sub_i32_constant:
1683; GFX1032:       ; %bb.0: ; %entry
1684; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1685; GFX1032-NEXT:    s_mov_b32 s2, exec_lo
1686; GFX1032-NEXT:    ; implicit-def: $vgpr1
1687; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
1688; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
1689; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
1690; GFX1032-NEXT:    s_cbranch_execz BB8_2
1691; GFX1032-NEXT:  ; %bb.1:
1692; GFX1032-NEXT:    s_bcnt1_i32_b32 s2, s2
1693; GFX1032-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
1694; GFX1032-NEXT:    v_mul_u32_u24_e64 v1, s2, 5
1695; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1696; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
1697; GFX1032-NEXT:    ds_sub_rtn_u32 v1, v2, v1
1698; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1699; GFX1032-NEXT:    buffer_gl0_inv
1700; GFX1032-NEXT:    buffer_gl1_inv
1701; GFX1032-NEXT:  BB8_2:
1702; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
1703; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s3
1704; GFX1032-NEXT:    v_readfirstlane_b32 s2, v1
1705; GFX1032-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
1706; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
1707; GFX1032-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
1708; GFX1032-NEXT:    s_mov_b32 s2, -1
1709; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
1710; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1711; GFX1032-NEXT:    s_endpgm
1712entry:
1713  %old = atomicrmw sub i32 addrspace(3)* @local_var32, i32 5 acq_rel
1714  store i32 %old, i32 addrspace(1)* %out
1715  ret void
1716}
1717
1718define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive) {
1719;
1720;
1721; GFX7LESS-LABEL: sub_i32_uniform:
1722; GFX7LESS:       ; %bb.0: ; %entry
1723; GFX7LESS-NEXT:    s_mov_b64 s[6:7], exec
1724; GFX7LESS-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1725; GFX7LESS-NEXT:    s_load_dword s2, s[0:1], 0xb
1726; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
1727; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s7, v0
1728; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
1729; GFX7LESS-NEXT:    ; implicit-def: $vgpr1
1730; GFX7LESS-NEXT:    s_and_saveexec_b64 s[0:1], vcc
1731; GFX7LESS-NEXT:    s_cbranch_execz BB9_2
1732; GFX7LESS-NEXT:  ; %bb.1:
1733; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s3, s[6:7]
1734; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
1735; GFX7LESS-NEXT:    s_mul_i32 s3, s2, s3
1736; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
1737; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s3
1738; GFX7LESS-NEXT:    s_mov_b32 m0, -1
1739; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1740; GFX7LESS-NEXT:    ds_sub_rtn_u32 v1, v1, v2
1741; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1742; GFX7LESS-NEXT:    buffer_wbinvl1
1743; GFX7LESS-NEXT:  BB9_2:
1744; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[0:1]
1745; GFX7LESS-NEXT:    v_readfirstlane_b32 s0, v1
1746; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
1747; GFX7LESS-NEXT:    v_mul_lo_u32 v0, s2, v0
1748; GFX7LESS-NEXT:    s_mov_b32 s7, 0xf000
1749; GFX7LESS-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
1750; GFX7LESS-NEXT:    s_mov_b32 s6, -1
1751; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1752; GFX7LESS-NEXT:    s_endpgm
1753;
1754; GFX8-LABEL: sub_i32_uniform:
1755; GFX8:       ; %bb.0: ; %entry
1756; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1757; GFX8-NEXT:    s_load_dword s0, s[0:1], 0x2c
1758; GFX8-NEXT:    s_mov_b64 s[2:3], exec
1759; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
1760; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
1761; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
1762; GFX8-NEXT:    ; implicit-def: $vgpr1
1763; GFX8-NEXT:    s_and_saveexec_b64 s[6:7], vcc
1764; GFX8-NEXT:    s_cbranch_execz BB9_2
1765; GFX8-NEXT:  ; %bb.1:
1766; GFX8-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
1767; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1768; GFX8-NEXT:    s_mul_i32 s1, s0, s1
1769; GFX8-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
1770; GFX8-NEXT:    v_mov_b32_e32 v2, s1
1771; GFX8-NEXT:    s_mov_b32 m0, -1
1772; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1773; GFX8-NEXT:    ds_sub_rtn_u32 v1, v1, v2
1774; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1775; GFX8-NEXT:    buffer_wbinvl1_vol
1776; GFX8-NEXT:  BB9_2:
1777; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]
1778; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1779; GFX8-NEXT:    v_mul_lo_u32 v0, s0, v0
1780; GFX8-NEXT:    v_readfirstlane_b32 s0, v1
1781; GFX8-NEXT:    s_mov_b32 s7, 0xf000
1782; GFX8-NEXT:    s_mov_b32 s6, -1
1783; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s0, v0
1784; GFX8-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1785; GFX8-NEXT:    s_endpgm
1786;
1787; GFX9-LABEL: sub_i32_uniform:
1788; GFX9:       ; %bb.0: ; %entry
1789; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1790; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x2c
1791; GFX9-NEXT:    s_mov_b64 s[2:3], exec
1792; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
1793; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
1794; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
1795; GFX9-NEXT:    ; implicit-def: $vgpr1
1796; GFX9-NEXT:    s_and_saveexec_b64 s[6:7], vcc
1797; GFX9-NEXT:    s_cbranch_execz BB9_2
1798; GFX9-NEXT:  ; %bb.1:
1799; GFX9-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
1800; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1801; GFX9-NEXT:    s_mul_i32 s1, s0, s1
1802; GFX9-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
1803; GFX9-NEXT:    v_mov_b32_e32 v2, s1
1804; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1805; GFX9-NEXT:    ds_sub_rtn_u32 v1, v1, v2
1806; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1807; GFX9-NEXT:    buffer_wbinvl1_vol
1808; GFX9-NEXT:  BB9_2:
1809; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]
1810; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1811; GFX9-NEXT:    v_mul_lo_u32 v0, s0, v0
1812; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
1813; GFX9-NEXT:    s_mov_b32 s7, 0xf000
1814; GFX9-NEXT:    s_mov_b32 s6, -1
1815; GFX9-NEXT:    v_sub_u32_e32 v0, s0, v0
1816; GFX9-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1817; GFX9-NEXT:    s_endpgm
1818;
1819; GFX1064-LABEL: sub_i32_uniform:
1820; GFX1064:       ; %bb.0: ; %entry
1821; GFX1064-NEXT:    s_clause 0x1
1822; GFX1064-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1823; GFX1064-NEXT:    s_load_dword s0, s[0:1], 0x2c
1824; GFX1064-NEXT:    s_mov_b64 s[2:3], exec
1825; GFX1064-NEXT:    ; implicit-def: $vgpr1
1826; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
1827; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
1828; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
1829; GFX1064-NEXT:    s_and_saveexec_b64 s[6:7], vcc
1830; GFX1064-NEXT:    s_cbranch_execz BB9_2
1831; GFX1064-NEXT:  ; %bb.1:
1832; GFX1064-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
1833; GFX1064-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
1834; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
1835; GFX1064-NEXT:    s_mul_i32 s1, s0, s1
1836; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
1837; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1838; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
1839; GFX1064-NEXT:    ds_sub_rtn_u32 v1, v1, v2
1840; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1841; GFX1064-NEXT:    buffer_gl0_inv
1842; GFX1064-NEXT:    buffer_gl1_inv
1843; GFX1064-NEXT:  BB9_2:
1844; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
1845; GFX1064-NEXT:    s_or_b64 exec, exec, s[6:7]
1846; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
1847; GFX1064-NEXT:    v_mul_lo_u32 v0, s0, v0
1848; GFX1064-NEXT:    v_readfirstlane_b32 s0, v1
1849; GFX1064-NEXT:    s_mov_b32 s7, 0x31016000
1850; GFX1064-NEXT:    s_mov_b32 s6, -1
1851; GFX1064-NEXT:    v_sub_nc_u32_e32 v0, s0, v0
1852; GFX1064-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1853; GFX1064-NEXT:    s_endpgm
1854;
1855; GFX1032-LABEL: sub_i32_uniform:
1856; GFX1032:       ; %bb.0: ; %entry
1857; GFX1032-NEXT:    s_clause 0x1
1858; GFX1032-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1859; GFX1032-NEXT:    s_load_dword s0, s[0:1], 0x2c
1860; GFX1032-NEXT:    s_mov_b32 s2, exec_lo
1861; GFX1032-NEXT:    ; implicit-def: $vgpr1
1862; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
1863; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
1864; GFX1032-NEXT:    s_and_saveexec_b32 s1, vcc_lo
1865; GFX1032-NEXT:    s_cbranch_execz BB9_2
1866; GFX1032-NEXT:  ; %bb.1:
1867; GFX1032-NEXT:    s_bcnt1_i32_b32 s2, s2
1868; GFX1032-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
1869; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
1870; GFX1032-NEXT:    s_mul_i32 s2, s0, s2
1871; GFX1032-NEXT:    v_mov_b32_e32 v2, s2
1872; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1873; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
1874; GFX1032-NEXT:    ds_sub_rtn_u32 v1, v1, v2
1875; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1876; GFX1032-NEXT:    buffer_gl0_inv
1877; GFX1032-NEXT:    buffer_gl1_inv
1878; GFX1032-NEXT:  BB9_2:
1879; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
1880; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s1
1881; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
1882; GFX1032-NEXT:    v_mul_lo_u32 v0, s0, v0
1883; GFX1032-NEXT:    v_readfirstlane_b32 s0, v1
1884; GFX1032-NEXT:    s_mov_b32 s7, 0x31016000
1885; GFX1032-NEXT:    s_mov_b32 s6, -1
1886; GFX1032-NEXT:    v_sub_nc_u32_e32 v0, s0, v0
1887; GFX1032-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1888; GFX1032-NEXT:    s_endpgm
1889entry:
1890  %old = atomicrmw sub i32 addrspace(3)* @local_var32, i32 %subitive acq_rel
1891  store i32 %old, i32 addrspace(1)* %out
1892  ret void
1893}
1894
1895define amdgpu_kernel void @sub_i32_varying(i32 addrspace(1)* %out) {
1896;
1897;
1898; GFX7LESS-LABEL: sub_i32_varying:
1899; GFX7LESS:       ; %bb.0: ; %entry
1900; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
1901; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
1902; GFX7LESS-NEXT:    s_mov_b32 m0, -1
1903; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1904; GFX7LESS-NEXT:    ds_sub_rtn_u32 v0, v1, v0
1905; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1906; GFX7LESS-NEXT:    buffer_wbinvl1
1907; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
1908; GFX7LESS-NEXT:    s_mov_b32 s2, -1
1909; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1910; GFX7LESS-NEXT:    s_endpgm
1911;
1912; GFX8-LABEL: sub_i32_varying:
1913; GFX8:       ; %bb.0: ; %entry
1914; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1915; GFX8-NEXT:    v_mov_b32_e32 v2, v0
1916; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
1917; GFX8-NEXT:    v_mov_b32_e32 v1, 0
1918; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
1919; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
1920; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
1921; GFX8-NEXT:    s_not_b64 exec, exec
1922; GFX8-NEXT:    v_mov_b32_e32 v2, 0
1923; GFX8-NEXT:    s_not_b64 exec, exec
1924; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
1925; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
1926; GFX8-NEXT:    s_nop 1
1927; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
1928; GFX8-NEXT:    s_nop 1
1929; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
1930; GFX8-NEXT:    s_nop 1
1931; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
1932; GFX8-NEXT:    s_nop 1
1933; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
1934; GFX8-NEXT:    s_nop 1
1935; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
1936; GFX8-NEXT:    v_readlane_b32 s4, v2, 63
1937; GFX8-NEXT:    s_nop 0
1938; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
1939; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
1940; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
1941; GFX8-NEXT:    ; implicit-def: $vgpr0
1942; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
1943; GFX8-NEXT:    s_cbranch_execz BB10_2
1944; GFX8-NEXT:  ; %bb.1:
1945; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
1946; GFX8-NEXT:    v_mov_b32_e32 v3, s4
1947; GFX8-NEXT:    s_mov_b32 m0, -1
1948; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1949; GFX8-NEXT:    ds_sub_rtn_u32 v0, v0, v3
1950; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1951; GFX8-NEXT:    buffer_wbinvl1_vol
1952; GFX8-NEXT:  BB10_2:
1953; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
1954; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
1955; GFX8-NEXT:    v_mov_b32_e32 v0, v1
1956; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s2, v0
1957; GFX8-NEXT:    s_mov_b32 s3, 0xf000
1958; GFX8-NEXT:    s_mov_b32 s2, -1
1959; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
1960; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1961; GFX8-NEXT:    s_endpgm
1962;
1963; GFX9-LABEL: sub_i32_varying:
1964; GFX9:       ; %bb.0: ; %entry
1965; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1966; GFX9-NEXT:    v_mov_b32_e32 v2, v0
1967; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
1968; GFX9-NEXT:    v_mov_b32_e32 v1, 0
1969; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
1970; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
1971; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
1972; GFX9-NEXT:    s_not_b64 exec, exec
1973; GFX9-NEXT:    v_mov_b32_e32 v2, 0
1974; GFX9-NEXT:    s_not_b64 exec, exec
1975; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
1976; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
1977; GFX9-NEXT:    s_nop 1
1978; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
1979; GFX9-NEXT:    s_nop 1
1980; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
1981; GFX9-NEXT:    s_nop 1
1982; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
1983; GFX9-NEXT:    s_nop 1
1984; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
1985; GFX9-NEXT:    s_nop 1
1986; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
1987; GFX9-NEXT:    v_readlane_b32 s4, v2, 63
1988; GFX9-NEXT:    s_nop 0
1989; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
1990; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
1991; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
1992; GFX9-NEXT:    ; implicit-def: $vgpr0
1993; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
1994; GFX9-NEXT:    s_cbranch_execz BB10_2
1995; GFX9-NEXT:  ; %bb.1:
1996; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
1997; GFX9-NEXT:    v_mov_b32_e32 v3, s4
1998; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1999; GFX9-NEXT:    ds_sub_rtn_u32 v0, v0, v3
2000; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2001; GFX9-NEXT:    buffer_wbinvl1_vol
2002; GFX9-NEXT:  BB10_2:
2003; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
2004; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
2005; GFX9-NEXT:    v_mov_b32_e32 v0, v1
2006; GFX9-NEXT:    v_sub_u32_e32 v0, s2, v0
2007; GFX9-NEXT:    s_mov_b32 s3, 0xf000
2008; GFX9-NEXT:    s_mov_b32 s2, -1
2009; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
2010; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2011; GFX9-NEXT:    s_endpgm
2012;
2013; GFX1064-LABEL: sub_i32_varying:
2014; GFX1064:       ; %bb.0: ; %entry
2015; GFX1064-NEXT:    v_mov_b32_e32 v1, v0
2016; GFX1064-NEXT:    s_not_b64 exec, exec
2017; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
2018; GFX1064-NEXT:    s_not_b64 exec, exec
2019; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
2020; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
2021; GFX1064-NEXT:    v_mov_b32_e32 v3, 0
2022; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
2023; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
2024; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
2025; GFX1064-NEXT:    v_mov_b32_e32 v2, v1
2026; GFX1064-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
2027; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
2028; GFX1064-NEXT:    v_readlane_b32 s4, v1, 31
2029; GFX1064-NEXT:    v_mov_b32_e32 v2, s4
2030; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
2031; GFX1064-NEXT:    v_readlane_b32 s4, v1, 15
2032; GFX1064-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
2033; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
2034; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
2035; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
2036; GFX1064-NEXT:    v_readlane_b32 s5, v1, 31
2037; GFX1064-NEXT:    v_writelane_b32 v3, s4, 16
2038; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
2039; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
2040; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
2041; GFX1064-NEXT:    v_readlane_b32 s7, v1, 63
2042; GFX1064-NEXT:    v_readlane_b32 s6, v1, 47
2043; GFX1064-NEXT:    v_writelane_b32 v3, s5, 32
2044; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
2045; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
2046; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
2047; GFX1064-NEXT:    v_writelane_b32 v3, s6, 48
2048; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
2049; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
2050; GFX1064-NEXT:    s_mov_b32 s2, -1
2051; GFX1064-NEXT:    ; implicit-def: $vgpr0
2052; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
2053; GFX1064-NEXT:    s_cbranch_execz BB10_2
2054; GFX1064-NEXT:  ; %bb.1:
2055; GFX1064-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
2056; GFX1064-NEXT:    v_mov_b32_e32 v4, s7
2057; GFX1064-NEXT:    s_mov_b32 s3, s7
2058; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2059; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
2060; GFX1064-NEXT:    ds_sub_rtn_u32 v0, v7, v4
2061; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2062; GFX1064-NEXT:    buffer_gl0_inv
2063; GFX1064-NEXT:    buffer_gl1_inv
2064; GFX1064-NEXT:  BB10_2:
2065; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
2066; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
2067; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
2068; GFX1064-NEXT:    v_mov_b32_e32 v0, v3
2069; GFX1064-NEXT:    v_sub_nc_u32_e32 v0, s3, v0
2070; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
2071; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
2072; GFX1064-NEXT:    s_nop 0
2073; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2074; GFX1064-NEXT:    s_endpgm
2075;
2076; GFX1032-LABEL: sub_i32_varying:
2077; GFX1032:       ; %bb.0: ; %entry
2078; GFX1032-NEXT:    v_mov_b32_e32 v1, v0
2079; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
2080; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
2081; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
2082; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
2083; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
2084; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
2085; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
2086; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
2087; GFX1032-NEXT:    v_mov_b32_e32 v2, v1
2088; GFX1032-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
2089; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
2090; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
2091; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
2092; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
2093; GFX1032-NEXT:    v_mov_b32_e32 v3, 0
2094; GFX1032-NEXT:    v_readlane_b32 s3, v1, 15
2095; GFX1032-NEXT:    v_readlane_b32 s4, v1, 31
2096; GFX1032-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
2097; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
2098; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
2099; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
2100; GFX1032-NEXT:    v_writelane_b32 v3, s3, 16
2101; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
2102; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
2103; GFX1032-NEXT:    s_mov_b32 s2, -1
2104; GFX1032-NEXT:    ; implicit-def: $vgpr0
2105; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
2106; GFX1032-NEXT:    s_cbranch_execz BB10_2
2107; GFX1032-NEXT:  ; %bb.1:
2108; GFX1032-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
2109; GFX1032-NEXT:    v_mov_b32_e32 v4, s4
2110; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2111; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
2112; GFX1032-NEXT:    ds_sub_rtn_u32 v0, v7, v4
2113; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2114; GFX1032-NEXT:    buffer_gl0_inv
2115; GFX1032-NEXT:    buffer_gl1_inv
2116; GFX1032-NEXT:  BB10_2:
2117; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
2118; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s3
2119; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
2120; GFX1032-NEXT:    v_mov_b32_e32 v0, v3
2121; GFX1032-NEXT:    v_sub_nc_u32_e32 v0, s3, v0
2122; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
2123; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
2124; GFX1032-NEXT:    s_nop 0
2125; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2126; GFX1032-NEXT:    s_endpgm
2127entry:
2128  %lane = call i32 @llvm.amdgcn.workitem.id.x()
2129  %old = atomicrmw sub i32 addrspace(3)* @local_var32, i32 %lane acq_rel
2130  store i32 %old, i32 addrspace(1)* %out
2131  ret void
2132}
2133
2134define amdgpu_kernel void @sub_i64_constant(i64 addrspace(1)* %out) {
2135;
2136;
2137; GFX7LESS-LABEL: sub_i64_constant:
2138; GFX7LESS:       ; %bb.0: ; %entry
2139; GFX7LESS-NEXT:    s_mov_b64 s[4:5], exec
2140; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
2141; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
2142; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s5, v0
2143; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
2144; GFX7LESS-NEXT:    ; implicit-def: $vgpr1_vgpr2
2145; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
2146; GFX7LESS-NEXT:    s_cbranch_execz BB11_2
2147; GFX7LESS-NEXT:  ; %bb.1:
2148; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
2149; GFX7LESS-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
2150; GFX7LESS-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
2151; GFX7LESS-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
2152; GFX7LESS-NEXT:    s_mov_b32 m0, -1
2153; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2154; GFX7LESS-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
2155; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2156; GFX7LESS-NEXT:    buffer_wbinvl1
2157; GFX7LESS-NEXT:  BB11_2:
2158; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
2159; GFX7LESS-NEXT:    v_readfirstlane_b32 s2, v1
2160; GFX7LESS-NEXT:    v_readfirstlane_b32 s4, v2
2161; GFX7LESS-NEXT:    v_mul_hi_u32_u24_e32 v1, 5, v0
2162; GFX7LESS-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
2163; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
2164; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s4
2165; GFX7LESS-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
2166; GFX7LESS-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
2167; GFX7LESS-NEXT:    s_mov_b32 s2, -1
2168; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
2169; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
2170; GFX7LESS-NEXT:    s_endpgm
2171;
2172; GFX8-LABEL: sub_i64_constant:
2173; GFX8:       ; %bb.0: ; %entry
2174; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
2175; GFX8-NEXT:    s_mov_b64 s[4:5], exec
2176; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
2177; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
2178; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
2179; GFX8-NEXT:    ; implicit-def: $vgpr1_vgpr2
2180; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
2181; GFX8-NEXT:    s_cbranch_execz BB11_2
2182; GFX8-NEXT:  ; %bb.1:
2183; GFX8-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
2184; GFX8-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
2185; GFX8-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
2186; GFX8-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
2187; GFX8-NEXT:    s_mov_b32 m0, -1
2188; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2189; GFX8-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
2190; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2191; GFX8-NEXT:    buffer_wbinvl1_vol
2192; GFX8-NEXT:  BB11_2:
2193; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
2194; GFX8-NEXT:    v_readfirstlane_b32 s3, v2
2195; GFX8-NEXT:    v_readfirstlane_b32 s2, v1
2196; GFX8-NEXT:    v_mul_hi_u32_u24_e32 v1, 5, v0
2197; GFX8-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
2198; GFX8-NEXT:    v_mov_b32_e32 v2, s3
2199; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s2, v0
2200; GFX8-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
2201; GFX8-NEXT:    s_mov_b32 s3, 0xf000
2202; GFX8-NEXT:    s_mov_b32 s2, -1
2203; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2204; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
2205; GFX8-NEXT:    s_endpgm
2206;
2207; GFX9-LABEL: sub_i64_constant:
2208; GFX9:       ; %bb.0: ; %entry
2209; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
2210; GFX9-NEXT:    s_mov_b64 s[4:5], exec
2211; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
2212; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
2213; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
2214; GFX9-NEXT:    ; implicit-def: $vgpr1_vgpr2
2215; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
2216; GFX9-NEXT:    s_cbranch_execz BB11_2
2217; GFX9-NEXT:  ; %bb.1:
2218; GFX9-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
2219; GFX9-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
2220; GFX9-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
2221; GFX9-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
2222; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2223; GFX9-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
2224; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2225; GFX9-NEXT:    buffer_wbinvl1_vol
2226; GFX9-NEXT:  BB11_2:
2227; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
2228; GFX9-NEXT:    v_readfirstlane_b32 s3, v2
2229; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
2230; GFX9-NEXT:    v_mul_hi_u32_u24_e32 v1, 5, v0
2231; GFX9-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
2232; GFX9-NEXT:    v_mov_b32_e32 v2, s3
2233; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, s2, v0
2234; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v2, v1, vcc
2235; GFX9-NEXT:    s_mov_b32 s3, 0xf000
2236; GFX9-NEXT:    s_mov_b32 s2, -1
2237; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
2238; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
2239; GFX9-NEXT:    s_endpgm
2240;
2241; GFX1064-LABEL: sub_i64_constant:
2242; GFX1064:       ; %bb.0: ; %entry
2243; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
2244; GFX1064-NEXT:    s_mov_b64 s[4:5], exec
2245; GFX1064-NEXT:    ; implicit-def: $vgpr1_vgpr2
2246; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
2247; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s5, v0
2248; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
2249; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
2250; GFX1064-NEXT:    s_cbranch_execz BB11_2
2251; GFX1064-NEXT:  ; %bb.1:
2252; GFX1064-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
2253; GFX1064-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
2254; GFX1064-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
2255; GFX1064-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
2256; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2257; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
2258; GFX1064-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
2259; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2260; GFX1064-NEXT:    buffer_gl0_inv
2261; GFX1064-NEXT:    buffer_gl1_inv
2262; GFX1064-NEXT:  BB11_2:
2263; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
2264; GFX1064-NEXT:    s_or_b64 exec, exec, s[2:3]
2265; GFX1064-NEXT:    v_readfirstlane_b32 s2, v1
2266; GFX1064-NEXT:    v_mul_u32_u24_e32 v1, 5, v0
2267; GFX1064-NEXT:    v_readfirstlane_b32 s3, v2
2268; GFX1064-NEXT:    v_mul_hi_u32_u24_e32 v2, 5, v0
2269; GFX1064-NEXT:    v_sub_co_u32_e64 v0, vcc, s2, v1
2270; GFX1064-NEXT:    s_mov_b32 s2, -1
2271; GFX1064-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s3, v2, vcc
2272; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
2273; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
2274; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
2275; GFX1064-NEXT:    s_endpgm
2276;
2277; GFX1032-LABEL: sub_i64_constant:
2278; GFX1032:       ; %bb.0: ; %entry
2279; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
2280; GFX1032-NEXT:    s_mov_b32 s3, exec_lo
2281; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
2282; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
2283; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
2284; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
2285; GFX1032-NEXT:    s_cbranch_execz BB11_2
2286; GFX1032-NEXT:  ; %bb.1:
2287; GFX1032-NEXT:    s_bcnt1_i32_b32 s3, s3
2288; GFX1032-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
2289; GFX1032-NEXT:    v_mul_hi_u32_u24_e64 v2, s3, 5
2290; GFX1032-NEXT:    v_mul_u32_u24_e64 v1, s3, 5
2291; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2292; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
2293; GFX1032-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
2294; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2295; GFX1032-NEXT:    buffer_gl0_inv
2296; GFX1032-NEXT:    buffer_gl1_inv
2297; GFX1032-NEXT:  BB11_2:
2298; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
2299; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2
2300; GFX1032-NEXT:    v_readfirstlane_b32 s2, v1
2301; GFX1032-NEXT:    v_mul_u32_u24_e32 v1, 5, v0
2302; GFX1032-NEXT:    v_readfirstlane_b32 s3, v2
2303; GFX1032-NEXT:    v_mul_hi_u32_u24_e32 v2, 5, v0
2304; GFX1032-NEXT:    v_sub_co_u32_e64 v0, vcc_lo, s2, v1
2305; GFX1032-NEXT:    s_mov_b32 s2, -1
2306; GFX1032-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s3, v2, vcc_lo
2307; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
2308; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
2309; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
2310; GFX1032-NEXT:    s_endpgm
2311entry:
2312  %old = atomicrmw sub i64 addrspace(3)* @local_var64, i64 5 acq_rel
2313  store i64 %old, i64 addrspace(1)* %out
2314  ret void
2315}
2316
2317define amdgpu_kernel void @sub_i64_uniform(i64 addrspace(1)* %out, i64 %subitive) {
2318;
2319;
2320; GFX7LESS-LABEL: sub_i64_uniform:
2321; GFX7LESS:       ; %bb.0: ; %entry
2322; GFX7LESS-NEXT:    s_mov_b64 s[6:7], exec
2323; GFX7LESS-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
2324; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
2325; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s7, v0
2326; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
2327; GFX7LESS-NEXT:    ; implicit-def: $vgpr1_vgpr2
2328; GFX7LESS-NEXT:    s_and_saveexec_b64 s[4:5], vcc
2329; GFX7LESS-NEXT:    s_cbranch_execz BB12_2
2330; GFX7LESS-NEXT:  ; %bb.1:
2331; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
2332; GFX7LESS-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
2333; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
2334; GFX7LESS-NEXT:    s_mul_i32 s7, s3, s6
2335; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s6
2336; GFX7LESS-NEXT:    v_mul_hi_u32 v1, s2, v1
2337; GFX7LESS-NEXT:    s_mul_i32 s6, s2, s6
2338; GFX7LESS-NEXT:    v_add_i32_e32 v2, vcc, s7, v1
2339; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s6
2340; GFX7LESS-NEXT:    s_mov_b32 m0, -1
2341; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2342; GFX7LESS-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
2343; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2344; GFX7LESS-NEXT:    buffer_wbinvl1
2345; GFX7LESS-NEXT:  BB12_2:
2346; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[4:5]
2347; GFX7LESS-NEXT:    s_mov_b32 s7, 0xf000
2348; GFX7LESS-NEXT:    s_mov_b32 s6, -1
2349; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
2350; GFX7LESS-NEXT:    s_mov_b32 s4, s0
2351; GFX7LESS-NEXT:    s_mov_b32 s5, s1
2352; GFX7LESS-NEXT:    v_readfirstlane_b32 s0, v1
2353; GFX7LESS-NEXT:    v_readfirstlane_b32 s1, v2
2354; GFX7LESS-NEXT:    v_mul_lo_u32 v1, s3, v0
2355; GFX7LESS-NEXT:    v_mul_hi_u32 v2, s2, v0
2356; GFX7LESS-NEXT:    v_mul_lo_u32 v0, s2, v0
2357; GFX7LESS-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
2358; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s1
2359; GFX7LESS-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
2360; GFX7LESS-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
2361; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
2362; GFX7LESS-NEXT:    s_endpgm
2363;
2364; GFX8-LABEL: sub_i64_uniform:
2365; GFX8:       ; %bb.0: ; %entry
2366; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
2367; GFX8-NEXT:    s_mov_b64 s[6:7], exec
2368; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
2369; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
2370; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
2371; GFX8-NEXT:    ; implicit-def: $vgpr1_vgpr2
2372; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
2373; GFX8-NEXT:    s_cbranch_execz BB12_2
2374; GFX8-NEXT:  ; %bb.1:
2375; GFX8-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
2376; GFX8-NEXT:    v_mov_b32_e32 v1, s6
2377; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2378; GFX8-NEXT:    v_mul_hi_u32 v1, s2, v1
2379; GFX8-NEXT:    s_mul_i32 s7, s3, s6
2380; GFX8-NEXT:    s_mul_i32 s6, s2, s6
2381; GFX8-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
2382; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s7, v1
2383; GFX8-NEXT:    v_mov_b32_e32 v1, s6
2384; GFX8-NEXT:    s_mov_b32 m0, -1
2385; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2386; GFX8-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
2387; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2388; GFX8-NEXT:    buffer_wbinvl1_vol
2389; GFX8-NEXT:  BB12_2:
2390; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
2391; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2392; GFX8-NEXT:    s_mov_b32 s4, s0
2393; GFX8-NEXT:    v_readfirstlane_b32 s0, v1
2394; GFX8-NEXT:    v_mul_lo_u32 v1, s3, v0
2395; GFX8-NEXT:    v_mul_hi_u32 v3, s2, v0
2396; GFX8-NEXT:    v_mul_lo_u32 v0, s2, v0
2397; GFX8-NEXT:    s_mov_b32 s5, s1
2398; GFX8-NEXT:    v_readfirstlane_b32 s1, v2
2399; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v3, v1
2400; GFX8-NEXT:    v_mov_b32_e32 v2, s1
2401; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s0, v0
2402; GFX8-NEXT:    s_mov_b32 s7, 0xf000
2403; GFX8-NEXT:    s_mov_b32 s6, -1
2404; GFX8-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
2405; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
2406; GFX8-NEXT:    s_endpgm
2407;
2408; GFX9-LABEL: sub_i64_uniform:
2409; GFX9:       ; %bb.0: ; %entry
2410; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
2411; GFX9-NEXT:    s_mov_b64 s[6:7], exec
2412; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
2413; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
2414; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
2415; GFX9-NEXT:    ; implicit-def: $vgpr1_vgpr2
2416; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
2417; GFX9-NEXT:    s_cbranch_execz BB12_2
2418; GFX9-NEXT:  ; %bb.1:
2419; GFX9-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
2420; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
2421; GFX9-NEXT:    s_mul_i32 s7, s3, s6
2422; GFX9-NEXT:    s_mul_hi_u32 s8, s2, s6
2423; GFX9-NEXT:    s_add_i32 s8, s8, s7
2424; GFX9-NEXT:    s_mul_i32 s6, s2, s6
2425; GFX9-NEXT:    v_mov_b32_e32 v1, s6
2426; GFX9-NEXT:    v_mov_b32_e32 v2, s8
2427; GFX9-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
2428; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2429; GFX9-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
2430; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2431; GFX9-NEXT:    buffer_wbinvl1_vol
2432; GFX9-NEXT:  BB12_2:
2433; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
2434; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
2435; GFX9-NEXT:    v_mul_lo_u32 v3, s3, v0
2436; GFX9-NEXT:    v_mul_hi_u32 v4, s2, v0
2437; GFX9-NEXT:    v_mul_lo_u32 v0, s2, v0
2438; GFX9-NEXT:    s_mov_b32 s4, s0
2439; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
2440; GFX9-NEXT:    s_mov_b32 s5, s1
2441; GFX9-NEXT:    v_readfirstlane_b32 s1, v2
2442; GFX9-NEXT:    v_add_u32_e32 v1, v4, v3
2443; GFX9-NEXT:    v_mov_b32_e32 v2, s1
2444; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, s0, v0
2445; GFX9-NEXT:    s_mov_b32 s7, 0xf000
2446; GFX9-NEXT:    s_mov_b32 s6, -1
2447; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v2, v1, vcc
2448; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
2449; GFX9-NEXT:    s_endpgm
2450;
2451; GFX1064-LABEL: sub_i64_uniform:
2452; GFX1064:       ; %bb.0: ; %entry
2453; GFX1064-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
2454; GFX1064-NEXT:    s_mov_b64 s[6:7], exec
2455; GFX1064-NEXT:    ; implicit-def: $vgpr1_vgpr2
2456; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
2457; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s7, v0
2458; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
2459; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
2460; GFX1064-NEXT:    s_cbranch_execz BB12_2
2461; GFX1064-NEXT:  ; %bb.1:
2462; GFX1064-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
2463; GFX1064-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
2464; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
2465; GFX1064-NEXT:    s_mul_i32 s7, s3, s6
2466; GFX1064-NEXT:    s_mul_hi_u32 s8, s2, s6
2467; GFX1064-NEXT:    s_mul_i32 s6, s2, s6
2468; GFX1064-NEXT:    s_add_i32 s8, s8, s7
2469; GFX1064-NEXT:    v_mov_b32_e32 v1, s6
2470; GFX1064-NEXT:    v_mov_b32_e32 v2, s8
2471; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2472; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
2473; GFX1064-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
2474; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2475; GFX1064-NEXT:    buffer_gl0_inv
2476; GFX1064-NEXT:    buffer_gl1_inv
2477; GFX1064-NEXT:  BB12_2:
2478; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
2479; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
2480; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
2481; GFX1064-NEXT:    v_mul_lo_u32 v3, s3, v0
2482; GFX1064-NEXT:    v_mul_hi_u32 v4, s2, v0
2483; GFX1064-NEXT:    v_mul_lo_u32 v0, s2, v0
2484; GFX1064-NEXT:    v_readfirstlane_b32 s2, v1
2485; GFX1064-NEXT:    v_readfirstlane_b32 s4, v2
2486; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
2487; GFX1064-NEXT:    v_add_nc_u32_e32 v1, v4, v3
2488; GFX1064-NEXT:    v_sub_co_u32_e64 v0, vcc, s2, v0
2489; GFX1064-NEXT:    s_mov_b32 s2, -1
2490; GFX1064-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s4, v1, vcc
2491; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
2492; GFX1064-NEXT:    s_endpgm
2493;
2494; GFX1032-LABEL: sub_i64_uniform:
2495; GFX1032:       ; %bb.0: ; %entry
2496; GFX1032-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
2497; GFX1032-NEXT:    s_mov_b32 s5, exec_lo
2498; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
2499; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s5, 0
2500; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
2501; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
2502; GFX1032-NEXT:    s_cbranch_execz BB12_2
2503; GFX1032-NEXT:  ; %bb.1:
2504; GFX1032-NEXT:    s_bcnt1_i32_b32 s5, s5
2505; GFX1032-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
2506; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
2507; GFX1032-NEXT:    s_mul_i32 s6, s3, s5
2508; GFX1032-NEXT:    s_mul_hi_u32 s7, s2, s5
2509; GFX1032-NEXT:    s_mul_i32 s5, s2, s5
2510; GFX1032-NEXT:    s_add_i32 s7, s7, s6
2511; GFX1032-NEXT:    v_mov_b32_e32 v1, s5
2512; GFX1032-NEXT:    v_mov_b32_e32 v2, s7
2513; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2514; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
2515; GFX1032-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
2516; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2517; GFX1032-NEXT:    buffer_gl0_inv
2518; GFX1032-NEXT:    buffer_gl1_inv
2519; GFX1032-NEXT:  BB12_2:
2520; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
2521; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
2522; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
2523; GFX1032-NEXT:    v_mul_lo_u32 v3, s3, v0
2524; GFX1032-NEXT:    v_mul_hi_u32 v4, s2, v0
2525; GFX1032-NEXT:    v_mul_lo_u32 v0, s2, v0
2526; GFX1032-NEXT:    v_readfirstlane_b32 s2, v1
2527; GFX1032-NEXT:    v_readfirstlane_b32 s4, v2
2528; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
2529; GFX1032-NEXT:    v_add_nc_u32_e32 v1, v4, v3
2530; GFX1032-NEXT:    v_sub_co_u32_e64 v0, vcc_lo, s2, v0
2531; GFX1032-NEXT:    s_mov_b32 s2, -1
2532; GFX1032-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s4, v1, vcc_lo
2533; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
2534; GFX1032-NEXT:    s_endpgm
2535entry:
2536  %old = atomicrmw sub i64 addrspace(3)* @local_var64, i64 %subitive acq_rel
2537  store i64 %old, i64 addrspace(1)* %out
2538  ret void
2539}
2540
2541; GCN-NOT: v_mbcnt_lo_u32_b32
2542; GCN-NOT: v_mbcnt_hi_u32_b32
2543; GCN-NOT: s_bcnt1_i32_b64
2544define amdgpu_kernel void @sub_i64_varying(i64 addrspace(1)* %out) {
2545;
2546;
2547; GFX7LESS-LABEL: sub_i64_varying:
2548; GFX7LESS:       ; %bb.0: ; %entry
2549; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
2550; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
2551; GFX7LESS-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
2552; GFX7LESS-NEXT:    s_mov_b32 m0, -1
2553; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2554; GFX7LESS-NEXT:    ds_sub_rtn_u64 v[0:1], v2, v[0:1]
2555; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2556; GFX7LESS-NEXT:    buffer_wbinvl1
2557; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
2558; GFX7LESS-NEXT:    s_mov_b32 s2, -1
2559; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
2560; GFX7LESS-NEXT:    s_endpgm
2561;
2562; GFX8-LABEL: sub_i64_varying:
2563; GFX8:       ; %bb.0: ; %entry
2564; GFX8-NEXT:    v_mov_b32_e32 v1, 0
2565; GFX8-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
2566; GFX8-NEXT:    s_mov_b32 m0, -1
2567; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
2568; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2569; GFX8-NEXT:    ds_sub_rtn_u64 v[0:1], v2, v[0:1]
2570; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2571; GFX8-NEXT:    buffer_wbinvl1_vol
2572; GFX8-NEXT:    s_mov_b32 s3, 0xf000
2573; GFX8-NEXT:    s_mov_b32 s2, -1
2574; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
2575; GFX8-NEXT:    s_endpgm
2576;
2577; GFX9-LABEL: sub_i64_varying:
2578; GFX9:       ; %bb.0: ; %entry
2579; GFX9-NEXT:    v_mov_b32_e32 v1, 0
2580; GFX9-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
2581; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
2582; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2583; GFX9-NEXT:    ds_sub_rtn_u64 v[0:1], v2, v[0:1]
2584; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2585; GFX9-NEXT:    buffer_wbinvl1_vol
2586; GFX9-NEXT:    s_mov_b32 s3, 0xf000
2587; GFX9-NEXT:    s_mov_b32 s2, -1
2588; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
2589; GFX9-NEXT:    s_endpgm
2590;
2591; GFX1064-LABEL: sub_i64_varying:
2592; GFX1064:       ; %bb.0: ; %entry
2593; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
2594; GFX1064-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
2595; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
2596; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
2597; GFX1064-NEXT:    s_mov_b32 s2, -1
2598; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2599; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
2600; GFX1064-NEXT:    ds_sub_rtn_u64 v[0:1], v2, v[0:1]
2601; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2602; GFX1064-NEXT:    buffer_gl0_inv
2603; GFX1064-NEXT:    buffer_gl1_inv
2604; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
2605; GFX1064-NEXT:    s_endpgm
2606;
2607; GFX1032-LABEL: sub_i64_varying:
2608; GFX1032:       ; %bb.0: ; %entry
2609; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
2610; GFX1032-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
2611; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
2612; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
2613; GFX1032-NEXT:    s_mov_b32 s2, -1
2614; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2615; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
2616; GFX1032-NEXT:    ds_sub_rtn_u64 v[0:1], v2, v[0:1]
2617; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2618; GFX1032-NEXT:    buffer_gl0_inv
2619; GFX1032-NEXT:    buffer_gl1_inv
2620; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
2621; GFX1032-NEXT:    s_endpgm
2622entry:
2623  %lane = call i32 @llvm.amdgcn.workitem.id.x()
2624  %zext = zext i32 %lane to i64
2625  %old = atomicrmw sub i64 addrspace(3)* @local_var64, i64 %zext acq_rel
2626  store i64 %old, i64 addrspace(1)* %out
2627  ret void
2628}
2629
2630define amdgpu_kernel void @and_i32_varying(i32 addrspace(1)* %out) {
2631;
2632;
2633; GFX7LESS-LABEL: and_i32_varying:
2634; GFX7LESS:       ; %bb.0: ; %entry
2635; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
2636; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
2637; GFX7LESS-NEXT:    s_mov_b32 m0, -1
2638; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2639; GFX7LESS-NEXT:    ds_and_rtn_b32 v0, v1, v0
2640; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2641; GFX7LESS-NEXT:    buffer_wbinvl1
2642; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
2643; GFX7LESS-NEXT:    s_mov_b32 s2, -1
2644; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2645; GFX7LESS-NEXT:    s_endpgm
2646;
2647; GFX8-LABEL: and_i32_varying:
2648; GFX8:       ; %bb.0: ; %entry
2649; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
2650; GFX8-NEXT:    v_mov_b32_e32 v2, v0
2651; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
2652; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
2653; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
2654; GFX8-NEXT:    v_mov_b32_e32 v1, -1
2655; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
2656; GFX8-NEXT:    s_not_b64 exec, exec
2657; GFX8-NEXT:    v_mov_b32_e32 v2, -1
2658; GFX8-NEXT:    s_not_b64 exec, exec
2659; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
2660; GFX8-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
2661; GFX8-NEXT:    s_nop 1
2662; GFX8-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
2663; GFX8-NEXT:    s_nop 1
2664; GFX8-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
2665; GFX8-NEXT:    s_nop 1
2666; GFX8-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
2667; GFX8-NEXT:    s_nop 1
2668; GFX8-NEXT:    v_and_b32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
2669; GFX8-NEXT:    s_nop 1
2670; GFX8-NEXT:    v_and_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
2671; GFX8-NEXT:    v_readlane_b32 s4, v2, 63
2672; GFX8-NEXT:    s_nop 0
2673; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
2674; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
2675; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
2676; GFX8-NEXT:    ; implicit-def: $vgpr0
2677; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
2678; GFX8-NEXT:    s_cbranch_execz BB14_2
2679; GFX8-NEXT:  ; %bb.1:
2680; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
2681; GFX8-NEXT:    v_mov_b32_e32 v3, s4
2682; GFX8-NEXT:    s_mov_b32 m0, -1
2683; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2684; GFX8-NEXT:    ds_and_rtn_b32 v0, v0, v3
2685; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2686; GFX8-NEXT:    buffer_wbinvl1_vol
2687; GFX8-NEXT:  BB14_2:
2688; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
2689; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
2690; GFX8-NEXT:    v_mov_b32_e32 v0, v1
2691; GFX8-NEXT:    v_and_b32_e32 v0, s2, v0
2692; GFX8-NEXT:    s_mov_b32 s3, 0xf000
2693; GFX8-NEXT:    s_mov_b32 s2, -1
2694; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2695; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2696; GFX8-NEXT:    s_endpgm
2697;
2698; GFX9-LABEL: and_i32_varying:
2699; GFX9:       ; %bb.0: ; %entry
2700; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
2701; GFX9-NEXT:    v_mov_b32_e32 v2, v0
2702; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
2703; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
2704; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
2705; GFX9-NEXT:    v_mov_b32_e32 v1, -1
2706; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
2707; GFX9-NEXT:    s_not_b64 exec, exec
2708; GFX9-NEXT:    v_mov_b32_e32 v2, -1
2709; GFX9-NEXT:    s_not_b64 exec, exec
2710; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
2711; GFX9-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
2712; GFX9-NEXT:    s_nop 1
2713; GFX9-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
2714; GFX9-NEXT:    s_nop 1
2715; GFX9-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
2716; GFX9-NEXT:    s_nop 1
2717; GFX9-NEXT:    v_and_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
2718; GFX9-NEXT:    s_nop 1
2719; GFX9-NEXT:    v_and_b32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
2720; GFX9-NEXT:    s_nop 1
2721; GFX9-NEXT:    v_and_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
2722; GFX9-NEXT:    v_readlane_b32 s4, v2, 63
2723; GFX9-NEXT:    s_nop 0
2724; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
2725; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
2726; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
2727; GFX9-NEXT:    ; implicit-def: $vgpr0
2728; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
2729; GFX9-NEXT:    s_cbranch_execz BB14_2
2730; GFX9-NEXT:  ; %bb.1:
2731; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
2732; GFX9-NEXT:    v_mov_b32_e32 v3, s4
2733; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2734; GFX9-NEXT:    ds_and_rtn_b32 v0, v0, v3
2735; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2736; GFX9-NEXT:    buffer_wbinvl1_vol
2737; GFX9-NEXT:  BB14_2:
2738; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
2739; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
2740; GFX9-NEXT:    v_mov_b32_e32 v0, v1
2741; GFX9-NEXT:    v_and_b32_e32 v0, s2, v0
2742; GFX9-NEXT:    s_mov_b32 s3, 0xf000
2743; GFX9-NEXT:    s_mov_b32 s2, -1
2744; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
2745; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2746; GFX9-NEXT:    s_endpgm
2747;
2748; GFX1064-LABEL: and_i32_varying:
2749; GFX1064:       ; %bb.0: ; %entry
2750; GFX1064-NEXT:    v_mov_b32_e32 v1, v0
2751; GFX1064-NEXT:    s_not_b64 exec, exec
2752; GFX1064-NEXT:    v_mov_b32_e32 v1, -1
2753; GFX1064-NEXT:    s_not_b64 exec, exec
2754; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
2755; GFX1064-NEXT:    v_and_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
2756; GFX1064-NEXT:    v_mov_b32_e32 v3, -1
2757; GFX1064-NEXT:    v_and_b32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
2758; GFX1064-NEXT:    v_and_b32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
2759; GFX1064-NEXT:    v_and_b32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
2760; GFX1064-NEXT:    v_mov_b32_e32 v2, v1
2761; GFX1064-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
2762; GFX1064-NEXT:    v_and_b32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
2763; GFX1064-NEXT:    v_readlane_b32 s4, v1, 31
2764; GFX1064-NEXT:    v_mov_b32_e32 v2, s4
2765; GFX1064-NEXT:    v_and_b32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
2766; GFX1064-NEXT:    v_readlane_b32 s4, v1, 15
2767; GFX1064-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
2768; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
2769; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
2770; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
2771; GFX1064-NEXT:    v_readlane_b32 s5, v1, 31
2772; GFX1064-NEXT:    v_writelane_b32 v3, s4, 16
2773; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
2774; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
2775; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
2776; GFX1064-NEXT:    v_readlane_b32 s7, v1, 63
2777; GFX1064-NEXT:    v_readlane_b32 s6, v1, 47
2778; GFX1064-NEXT:    v_writelane_b32 v3, s5, 32
2779; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
2780; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
2781; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
2782; GFX1064-NEXT:    v_writelane_b32 v3, s6, 48
2783; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
2784; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
2785; GFX1064-NEXT:    s_mov_b32 s2, -1
2786; GFX1064-NEXT:    ; implicit-def: $vgpr0
2787; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
2788; GFX1064-NEXT:    s_cbranch_execz BB14_2
2789; GFX1064-NEXT:  ; %bb.1:
2790; GFX1064-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
2791; GFX1064-NEXT:    v_mov_b32_e32 v4, s7
2792; GFX1064-NEXT:    s_mov_b32 s3, s7
2793; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2794; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
2795; GFX1064-NEXT:    ds_and_rtn_b32 v0, v7, v4
2796; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2797; GFX1064-NEXT:    buffer_gl0_inv
2798; GFX1064-NEXT:    buffer_gl1_inv
2799; GFX1064-NEXT:  BB14_2:
2800; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
2801; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
2802; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
2803; GFX1064-NEXT:    v_mov_b32_e32 v0, v3
2804; GFX1064-NEXT:    v_and_b32_e32 v0, s3, v0
2805; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
2806; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
2807; GFX1064-NEXT:    s_nop 0
2808; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2809; GFX1064-NEXT:    s_endpgm
2810;
2811; GFX1032-LABEL: and_i32_varying:
2812; GFX1032:       ; %bb.0: ; %entry
2813; GFX1032-NEXT:    v_mov_b32_e32 v1, v0
2814; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
2815; GFX1032-NEXT:    v_mov_b32_e32 v1, -1
2816; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
2817; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
2818; GFX1032-NEXT:    v_and_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
2819; GFX1032-NEXT:    v_and_b32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
2820; GFX1032-NEXT:    v_and_b32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
2821; GFX1032-NEXT:    v_and_b32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
2822; GFX1032-NEXT:    v_mov_b32_e32 v2, v1
2823; GFX1032-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
2824; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
2825; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
2826; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
2827; GFX1032-NEXT:    v_and_b32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
2828; GFX1032-NEXT:    v_mov_b32_e32 v3, -1
2829; GFX1032-NEXT:    v_readlane_b32 s3, v1, 15
2830; GFX1032-NEXT:    v_readlane_b32 s4, v1, 31
2831; GFX1032-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
2832; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
2833; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
2834; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
2835; GFX1032-NEXT:    v_writelane_b32 v3, s3, 16
2836; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
2837; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
2838; GFX1032-NEXT:    s_mov_b32 s2, -1
2839; GFX1032-NEXT:    ; implicit-def: $vgpr0
2840; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
2841; GFX1032-NEXT:    s_cbranch_execz BB14_2
2842; GFX1032-NEXT:  ; %bb.1:
2843; GFX1032-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
2844; GFX1032-NEXT:    v_mov_b32_e32 v4, s4
2845; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2846; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
2847; GFX1032-NEXT:    ds_and_rtn_b32 v0, v7, v4
2848; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2849; GFX1032-NEXT:    buffer_gl0_inv
2850; GFX1032-NEXT:    buffer_gl1_inv
2851; GFX1032-NEXT:  BB14_2:
2852; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
2853; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s3
2854; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
2855; GFX1032-NEXT:    v_mov_b32_e32 v0, v3
2856; GFX1032-NEXT:    v_and_b32_e32 v0, s3, v0
2857; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
2858; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
2859; GFX1032-NEXT:    s_nop 0
2860; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2861; GFX1032-NEXT:    s_endpgm
2862entry:
2863  %lane = call i32 @llvm.amdgcn.workitem.id.x()
2864  %old = atomicrmw and i32 addrspace(3)* @local_var32, i32 %lane acq_rel
2865  store i32 %old, i32 addrspace(1)* %out
2866  ret void
2867}
2868
2869define amdgpu_kernel void @or_i32_varying(i32 addrspace(1)* %out) {
2870;
2871;
2872; GFX7LESS-LABEL: or_i32_varying:
2873; GFX7LESS:       ; %bb.0: ; %entry
2874; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
2875; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
2876; GFX7LESS-NEXT:    s_mov_b32 m0, -1
2877; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2878; GFX7LESS-NEXT:    ds_or_rtn_b32 v0, v1, v0
2879; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2880; GFX7LESS-NEXT:    buffer_wbinvl1
2881; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
2882; GFX7LESS-NEXT:    s_mov_b32 s2, -1
2883; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2884; GFX7LESS-NEXT:    s_endpgm
2885;
2886; GFX8-LABEL: or_i32_varying:
2887; GFX8:       ; %bb.0: ; %entry
2888; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
2889; GFX8-NEXT:    v_mov_b32_e32 v2, v0
2890; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
2891; GFX8-NEXT:    v_mov_b32_e32 v1, 0
2892; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
2893; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
2894; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
2895; GFX8-NEXT:    s_not_b64 exec, exec
2896; GFX8-NEXT:    v_mov_b32_e32 v2, 0
2897; GFX8-NEXT:    s_not_b64 exec, exec
2898; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
2899; GFX8-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
2900; GFX8-NEXT:    s_nop 1
2901; GFX8-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
2902; GFX8-NEXT:    s_nop 1
2903; GFX8-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
2904; GFX8-NEXT:    s_nop 1
2905; GFX8-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
2906; GFX8-NEXT:    s_nop 1
2907; GFX8-NEXT:    v_or_b32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
2908; GFX8-NEXT:    s_nop 1
2909; GFX8-NEXT:    v_or_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
2910; GFX8-NEXT:    v_readlane_b32 s4, v2, 63
2911; GFX8-NEXT:    s_nop 0
2912; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
2913; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
2914; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
2915; GFX8-NEXT:    ; implicit-def: $vgpr0
2916; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
2917; GFX8-NEXT:    s_cbranch_execz BB15_2
2918; GFX8-NEXT:  ; %bb.1:
2919; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
2920; GFX8-NEXT:    v_mov_b32_e32 v3, s4
2921; GFX8-NEXT:    s_mov_b32 m0, -1
2922; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2923; GFX8-NEXT:    ds_or_rtn_b32 v0, v0, v3
2924; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2925; GFX8-NEXT:    buffer_wbinvl1_vol
2926; GFX8-NEXT:  BB15_2:
2927; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
2928; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
2929; GFX8-NEXT:    v_mov_b32_e32 v0, v1
2930; GFX8-NEXT:    v_or_b32_e32 v0, s2, v0
2931; GFX8-NEXT:    s_mov_b32 s3, 0xf000
2932; GFX8-NEXT:    s_mov_b32 s2, -1
2933; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
2934; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2935; GFX8-NEXT:    s_endpgm
2936;
2937; GFX9-LABEL: or_i32_varying:
2938; GFX9:       ; %bb.0: ; %entry
2939; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
2940; GFX9-NEXT:    v_mov_b32_e32 v2, v0
2941; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
2942; GFX9-NEXT:    v_mov_b32_e32 v1, 0
2943; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
2944; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
2945; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
2946; GFX9-NEXT:    s_not_b64 exec, exec
2947; GFX9-NEXT:    v_mov_b32_e32 v2, 0
2948; GFX9-NEXT:    s_not_b64 exec, exec
2949; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
2950; GFX9-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
2951; GFX9-NEXT:    s_nop 1
2952; GFX9-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
2953; GFX9-NEXT:    s_nop 1
2954; GFX9-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
2955; GFX9-NEXT:    s_nop 1
2956; GFX9-NEXT:    v_or_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
2957; GFX9-NEXT:    s_nop 1
2958; GFX9-NEXT:    v_or_b32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
2959; GFX9-NEXT:    s_nop 1
2960; GFX9-NEXT:    v_or_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
2961; GFX9-NEXT:    v_readlane_b32 s4, v2, 63
2962; GFX9-NEXT:    s_nop 0
2963; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
2964; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
2965; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
2966; GFX9-NEXT:    ; implicit-def: $vgpr0
2967; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
2968; GFX9-NEXT:    s_cbranch_execz BB15_2
2969; GFX9-NEXT:  ; %bb.1:
2970; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
2971; GFX9-NEXT:    v_mov_b32_e32 v3, s4
2972; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2973; GFX9-NEXT:    ds_or_rtn_b32 v0, v0, v3
2974; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2975; GFX9-NEXT:    buffer_wbinvl1_vol
2976; GFX9-NEXT:  BB15_2:
2977; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
2978; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
2979; GFX9-NEXT:    v_mov_b32_e32 v0, v1
2980; GFX9-NEXT:    v_or_b32_e32 v0, s2, v0
2981; GFX9-NEXT:    s_mov_b32 s3, 0xf000
2982; GFX9-NEXT:    s_mov_b32 s2, -1
2983; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
2984; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2985; GFX9-NEXT:    s_endpgm
2986;
2987; GFX1064-LABEL: or_i32_varying:
2988; GFX1064:       ; %bb.0: ; %entry
2989; GFX1064-NEXT:    v_mov_b32_e32 v1, v0
2990; GFX1064-NEXT:    s_not_b64 exec, exec
2991; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
2992; GFX1064-NEXT:    s_not_b64 exec, exec
2993; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
2994; GFX1064-NEXT:    v_or_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
2995; GFX1064-NEXT:    v_mov_b32_e32 v3, 0
2996; GFX1064-NEXT:    v_or_b32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
2997; GFX1064-NEXT:    v_or_b32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
2998; GFX1064-NEXT:    v_or_b32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
2999; GFX1064-NEXT:    v_mov_b32_e32 v2, v1
3000; GFX1064-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
3001; GFX1064-NEXT:    v_or_b32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
3002; GFX1064-NEXT:    v_readlane_b32 s4, v1, 31
3003; GFX1064-NEXT:    v_mov_b32_e32 v2, s4
3004; GFX1064-NEXT:    v_or_b32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
3005; GFX1064-NEXT:    v_readlane_b32 s4, v1, 15
3006; GFX1064-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
3007; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
3008; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
3009; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
3010; GFX1064-NEXT:    v_readlane_b32 s5, v1, 31
3011; GFX1064-NEXT:    v_writelane_b32 v3, s4, 16
3012; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
3013; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
3014; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
3015; GFX1064-NEXT:    v_readlane_b32 s7, v1, 63
3016; GFX1064-NEXT:    v_readlane_b32 s6, v1, 47
3017; GFX1064-NEXT:    v_writelane_b32 v3, s5, 32
3018; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
3019; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
3020; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
3021; GFX1064-NEXT:    v_writelane_b32 v3, s6, 48
3022; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
3023; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
3024; GFX1064-NEXT:    s_mov_b32 s2, -1
3025; GFX1064-NEXT:    ; implicit-def: $vgpr0
3026; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
3027; GFX1064-NEXT:    s_cbranch_execz BB15_2
3028; GFX1064-NEXT:  ; %bb.1:
3029; GFX1064-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
3030; GFX1064-NEXT:    v_mov_b32_e32 v4, s7
3031; GFX1064-NEXT:    s_mov_b32 s3, s7
3032; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3033; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
3034; GFX1064-NEXT:    ds_or_rtn_b32 v0, v7, v4
3035; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3036; GFX1064-NEXT:    buffer_gl0_inv
3037; GFX1064-NEXT:    buffer_gl1_inv
3038; GFX1064-NEXT:  BB15_2:
3039; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
3040; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
3041; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
3042; GFX1064-NEXT:    v_mov_b32_e32 v0, v3
3043; GFX1064-NEXT:    v_or_b32_e32 v0, s3, v0
3044; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
3045; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
3046; GFX1064-NEXT:    s_nop 0
3047; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3048; GFX1064-NEXT:    s_endpgm
3049;
3050; GFX1032-LABEL: or_i32_varying:
3051; GFX1032:       ; %bb.0: ; %entry
3052; GFX1032-NEXT:    v_mov_b32_e32 v1, v0
3053; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
3054; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
3055; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
3056; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
3057; GFX1032-NEXT:    v_or_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
3058; GFX1032-NEXT:    v_or_b32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
3059; GFX1032-NEXT:    v_or_b32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
3060; GFX1032-NEXT:    v_or_b32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
3061; GFX1032-NEXT:    v_mov_b32_e32 v2, v1
3062; GFX1032-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
3063; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
3064; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
3065; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
3066; GFX1032-NEXT:    v_or_b32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
3067; GFX1032-NEXT:    v_mov_b32_e32 v3, 0
3068; GFX1032-NEXT:    v_readlane_b32 s3, v1, 15
3069; GFX1032-NEXT:    v_readlane_b32 s4, v1, 31
3070; GFX1032-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
3071; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
3072; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
3073; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
3074; GFX1032-NEXT:    v_writelane_b32 v3, s3, 16
3075; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
3076; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
3077; GFX1032-NEXT:    s_mov_b32 s2, -1
3078; GFX1032-NEXT:    ; implicit-def: $vgpr0
3079; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
3080; GFX1032-NEXT:    s_cbranch_execz BB15_2
3081; GFX1032-NEXT:  ; %bb.1:
3082; GFX1032-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
3083; GFX1032-NEXT:    v_mov_b32_e32 v4, s4
3084; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3085; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
3086; GFX1032-NEXT:    ds_or_rtn_b32 v0, v7, v4
3087; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3088; GFX1032-NEXT:    buffer_gl0_inv
3089; GFX1032-NEXT:    buffer_gl1_inv
3090; GFX1032-NEXT:  BB15_2:
3091; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
3092; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s3
3093; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
3094; GFX1032-NEXT:    v_mov_b32_e32 v0, v3
3095; GFX1032-NEXT:    v_or_b32_e32 v0, s3, v0
3096; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
3097; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
3098; GFX1032-NEXT:    s_nop 0
3099; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3100; GFX1032-NEXT:    s_endpgm
3101entry:
3102  %lane = call i32 @llvm.amdgcn.workitem.id.x()
3103  %old = atomicrmw or i32 addrspace(3)* @local_var32, i32 %lane acq_rel
3104  store i32 %old, i32 addrspace(1)* %out
3105  ret void
3106}
3107
3108define amdgpu_kernel void @xor_i32_varying(i32 addrspace(1)* %out) {
3109;
3110;
3111; GFX7LESS-LABEL: xor_i32_varying:
3112; GFX7LESS:       ; %bb.0: ; %entry
3113; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
3114; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
3115; GFX7LESS-NEXT:    s_mov_b32 m0, -1
3116; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3117; GFX7LESS-NEXT:    ds_xor_rtn_b32 v0, v1, v0
3118; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3119; GFX7LESS-NEXT:    buffer_wbinvl1
3120; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
3121; GFX7LESS-NEXT:    s_mov_b32 s2, -1
3122; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3123; GFX7LESS-NEXT:    s_endpgm
3124;
3125; GFX8-LABEL: xor_i32_varying:
3126; GFX8:       ; %bb.0: ; %entry
3127; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
3128; GFX8-NEXT:    v_mov_b32_e32 v2, v0
3129; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
3130; GFX8-NEXT:    v_mov_b32_e32 v1, 0
3131; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
3132; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
3133; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
3134; GFX8-NEXT:    s_not_b64 exec, exec
3135; GFX8-NEXT:    v_mov_b32_e32 v2, 0
3136; GFX8-NEXT:    s_not_b64 exec, exec
3137; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
3138; GFX8-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
3139; GFX8-NEXT:    s_nop 1
3140; GFX8-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
3141; GFX8-NEXT:    s_nop 1
3142; GFX8-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
3143; GFX8-NEXT:    s_nop 1
3144; GFX8-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
3145; GFX8-NEXT:    s_nop 1
3146; GFX8-NEXT:    v_xor_b32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
3147; GFX8-NEXT:    s_nop 1
3148; GFX8-NEXT:    v_xor_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
3149; GFX8-NEXT:    v_readlane_b32 s4, v2, 63
3150; GFX8-NEXT:    s_nop 0
3151; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
3152; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
3153; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
3154; GFX8-NEXT:    ; implicit-def: $vgpr0
3155; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
3156; GFX8-NEXT:    s_cbranch_execz BB16_2
3157; GFX8-NEXT:  ; %bb.1:
3158; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
3159; GFX8-NEXT:    v_mov_b32_e32 v3, s4
3160; GFX8-NEXT:    s_mov_b32 m0, -1
3161; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3162; GFX8-NEXT:    ds_xor_rtn_b32 v0, v0, v3
3163; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3164; GFX8-NEXT:    buffer_wbinvl1_vol
3165; GFX8-NEXT:  BB16_2:
3166; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
3167; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
3168; GFX8-NEXT:    v_mov_b32_e32 v0, v1
3169; GFX8-NEXT:    v_xor_b32_e32 v0, s2, v0
3170; GFX8-NEXT:    s_mov_b32 s3, 0xf000
3171; GFX8-NEXT:    s_mov_b32 s2, -1
3172; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
3173; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3174; GFX8-NEXT:    s_endpgm
3175;
3176; GFX9-LABEL: xor_i32_varying:
3177; GFX9:       ; %bb.0: ; %entry
3178; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
3179; GFX9-NEXT:    v_mov_b32_e32 v2, v0
3180; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
3181; GFX9-NEXT:    v_mov_b32_e32 v1, 0
3182; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
3183; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
3184; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
3185; GFX9-NEXT:    s_not_b64 exec, exec
3186; GFX9-NEXT:    v_mov_b32_e32 v2, 0
3187; GFX9-NEXT:    s_not_b64 exec, exec
3188; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
3189; GFX9-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
3190; GFX9-NEXT:    s_nop 1
3191; GFX9-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
3192; GFX9-NEXT:    s_nop 1
3193; GFX9-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
3194; GFX9-NEXT:    s_nop 1
3195; GFX9-NEXT:    v_xor_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
3196; GFX9-NEXT:    s_nop 1
3197; GFX9-NEXT:    v_xor_b32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
3198; GFX9-NEXT:    s_nop 1
3199; GFX9-NEXT:    v_xor_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
3200; GFX9-NEXT:    v_readlane_b32 s4, v2, 63
3201; GFX9-NEXT:    s_nop 0
3202; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
3203; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
3204; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
3205; GFX9-NEXT:    ; implicit-def: $vgpr0
3206; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
3207; GFX9-NEXT:    s_cbranch_execz BB16_2
3208; GFX9-NEXT:  ; %bb.1:
3209; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
3210; GFX9-NEXT:    v_mov_b32_e32 v3, s4
3211; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3212; GFX9-NEXT:    ds_xor_rtn_b32 v0, v0, v3
3213; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3214; GFX9-NEXT:    buffer_wbinvl1_vol
3215; GFX9-NEXT:  BB16_2:
3216; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
3217; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
3218; GFX9-NEXT:    v_mov_b32_e32 v0, v1
3219; GFX9-NEXT:    v_xor_b32_e32 v0, s2, v0
3220; GFX9-NEXT:    s_mov_b32 s3, 0xf000
3221; GFX9-NEXT:    s_mov_b32 s2, -1
3222; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
3223; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3224; GFX9-NEXT:    s_endpgm
3225;
3226; GFX1064-LABEL: xor_i32_varying:
3227; GFX1064:       ; %bb.0: ; %entry
3228; GFX1064-NEXT:    v_mov_b32_e32 v1, v0
3229; GFX1064-NEXT:    s_not_b64 exec, exec
3230; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
3231; GFX1064-NEXT:    s_not_b64 exec, exec
3232; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
3233; GFX1064-NEXT:    v_xor_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
3234; GFX1064-NEXT:    v_mov_b32_e32 v3, 0
3235; GFX1064-NEXT:    v_xor_b32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
3236; GFX1064-NEXT:    v_xor_b32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
3237; GFX1064-NEXT:    v_xor_b32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
3238; GFX1064-NEXT:    v_mov_b32_e32 v2, v1
3239; GFX1064-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
3240; GFX1064-NEXT:    v_xor_b32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
3241; GFX1064-NEXT:    v_readlane_b32 s4, v1, 31
3242; GFX1064-NEXT:    v_mov_b32_e32 v2, s4
3243; GFX1064-NEXT:    v_xor_b32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
3244; GFX1064-NEXT:    v_readlane_b32 s4, v1, 15
3245; GFX1064-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
3246; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
3247; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
3248; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
3249; GFX1064-NEXT:    v_readlane_b32 s5, v1, 31
3250; GFX1064-NEXT:    v_writelane_b32 v3, s4, 16
3251; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
3252; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
3253; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
3254; GFX1064-NEXT:    v_readlane_b32 s7, v1, 63
3255; GFX1064-NEXT:    v_readlane_b32 s6, v1, 47
3256; GFX1064-NEXT:    v_writelane_b32 v3, s5, 32
3257; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
3258; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
3259; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
3260; GFX1064-NEXT:    v_writelane_b32 v3, s6, 48
3261; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
3262; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
3263; GFX1064-NEXT:    s_mov_b32 s2, -1
3264; GFX1064-NEXT:    ; implicit-def: $vgpr0
3265; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
3266; GFX1064-NEXT:    s_cbranch_execz BB16_2
3267; GFX1064-NEXT:  ; %bb.1:
3268; GFX1064-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
3269; GFX1064-NEXT:    v_mov_b32_e32 v4, s7
3270; GFX1064-NEXT:    s_mov_b32 s3, s7
3271; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3272; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
3273; GFX1064-NEXT:    ds_xor_rtn_b32 v0, v7, v4
3274; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3275; GFX1064-NEXT:    buffer_gl0_inv
3276; GFX1064-NEXT:    buffer_gl1_inv
3277; GFX1064-NEXT:  BB16_2:
3278; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
3279; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
3280; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
3281; GFX1064-NEXT:    v_mov_b32_e32 v0, v3
3282; GFX1064-NEXT:    v_xor_b32_e32 v0, s3, v0
3283; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
3284; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
3285; GFX1064-NEXT:    s_nop 0
3286; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3287; GFX1064-NEXT:    s_endpgm
3288;
3289; GFX1032-LABEL: xor_i32_varying:
3290; GFX1032:       ; %bb.0: ; %entry
3291; GFX1032-NEXT:    v_mov_b32_e32 v1, v0
3292; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
3293; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
3294; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
3295; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
3296; GFX1032-NEXT:    v_xor_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
3297; GFX1032-NEXT:    v_xor_b32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
3298; GFX1032-NEXT:    v_xor_b32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
3299; GFX1032-NEXT:    v_xor_b32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
3300; GFX1032-NEXT:    v_mov_b32_e32 v2, v1
3301; GFX1032-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
3302; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
3303; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
3304; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
3305; GFX1032-NEXT:    v_xor_b32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
3306; GFX1032-NEXT:    v_mov_b32_e32 v3, 0
3307; GFX1032-NEXT:    v_readlane_b32 s3, v1, 15
3308; GFX1032-NEXT:    v_readlane_b32 s4, v1, 31
3309; GFX1032-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
3310; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
3311; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
3312; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
3313; GFX1032-NEXT:    v_writelane_b32 v3, s3, 16
3314; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
3315; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
3316; GFX1032-NEXT:    s_mov_b32 s2, -1
3317; GFX1032-NEXT:    ; implicit-def: $vgpr0
3318; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
3319; GFX1032-NEXT:    s_cbranch_execz BB16_2
3320; GFX1032-NEXT:  ; %bb.1:
3321; GFX1032-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
3322; GFX1032-NEXT:    v_mov_b32_e32 v4, s4
3323; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3324; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
3325; GFX1032-NEXT:    ds_xor_rtn_b32 v0, v7, v4
3326; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3327; GFX1032-NEXT:    buffer_gl0_inv
3328; GFX1032-NEXT:    buffer_gl1_inv
3329; GFX1032-NEXT:  BB16_2:
3330; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
3331; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s3
3332; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
3333; GFX1032-NEXT:    v_mov_b32_e32 v0, v3
3334; GFX1032-NEXT:    v_xor_b32_e32 v0, s3, v0
3335; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
3336; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
3337; GFX1032-NEXT:    s_nop 0
3338; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3339; GFX1032-NEXT:    s_endpgm
3340entry:
3341  %lane = call i32 @llvm.amdgcn.workitem.id.x()
3342  %old = atomicrmw xor i32 addrspace(3)* @local_var32, i32 %lane acq_rel
3343  store i32 %old, i32 addrspace(1)* %out
3344  ret void
3345}
3346
3347define amdgpu_kernel void @max_i32_varying(i32 addrspace(1)* %out) {
3348;
3349;
3350; GFX7LESS-LABEL: max_i32_varying:
3351; GFX7LESS:       ; %bb.0: ; %entry
3352; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
3353; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
3354; GFX7LESS-NEXT:    s_mov_b32 m0, -1
3355; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3356; GFX7LESS-NEXT:    ds_max_rtn_i32 v0, v1, v0
3357; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3358; GFX7LESS-NEXT:    buffer_wbinvl1
3359; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
3360; GFX7LESS-NEXT:    s_mov_b32 s2, -1
3361; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3362; GFX7LESS-NEXT:    s_endpgm
3363;
3364; GFX8-LABEL: max_i32_varying:
3365; GFX8:       ; %bb.0: ; %entry
3366; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
3367; GFX8-NEXT:    v_mov_b32_e32 v2, v0
3368; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
3369; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
3370; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
3371; GFX8-NEXT:    v_bfrev_b32_e32 v1, 1
3372; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
3373; GFX8-NEXT:    s_not_b64 exec, exec
3374; GFX8-NEXT:    v_mov_b32_e32 v2, v1
3375; GFX8-NEXT:    s_not_b64 exec, exec
3376; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
3377; GFX8-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
3378; GFX8-NEXT:    s_nop 1
3379; GFX8-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
3380; GFX8-NEXT:    s_nop 1
3381; GFX8-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
3382; GFX8-NEXT:    s_nop 1
3383; GFX8-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
3384; GFX8-NEXT:    s_nop 1
3385; GFX8-NEXT:    v_max_i32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
3386; GFX8-NEXT:    s_nop 1
3387; GFX8-NEXT:    v_max_i32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
3388; GFX8-NEXT:    v_readlane_b32 s4, v2, 63
3389; GFX8-NEXT:    s_nop 0
3390; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
3391; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
3392; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
3393; GFX8-NEXT:    ; implicit-def: $vgpr0
3394; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
3395; GFX8-NEXT:    s_cbranch_execz BB17_2
3396; GFX8-NEXT:  ; %bb.1:
3397; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
3398; GFX8-NEXT:    v_mov_b32_e32 v3, s4
3399; GFX8-NEXT:    s_mov_b32 m0, -1
3400; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3401; GFX8-NEXT:    ds_max_rtn_i32 v0, v0, v3
3402; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3403; GFX8-NEXT:    buffer_wbinvl1_vol
3404; GFX8-NEXT:  BB17_2:
3405; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
3406; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
3407; GFX8-NEXT:    v_mov_b32_e32 v0, v1
3408; GFX8-NEXT:    v_max_i32_e32 v0, s2, v0
3409; GFX8-NEXT:    s_mov_b32 s3, 0xf000
3410; GFX8-NEXT:    s_mov_b32 s2, -1
3411; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
3412; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3413; GFX8-NEXT:    s_endpgm
3414;
3415; GFX9-LABEL: max_i32_varying:
3416; GFX9:       ; %bb.0: ; %entry
3417; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
3418; GFX9-NEXT:    v_mov_b32_e32 v2, v0
3419; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
3420; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
3421; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
3422; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
3423; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
3424; GFX9-NEXT:    s_not_b64 exec, exec
3425; GFX9-NEXT:    v_mov_b32_e32 v2, v1
3426; GFX9-NEXT:    s_not_b64 exec, exec
3427; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
3428; GFX9-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
3429; GFX9-NEXT:    s_nop 1
3430; GFX9-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
3431; GFX9-NEXT:    s_nop 1
3432; GFX9-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
3433; GFX9-NEXT:    s_nop 1
3434; GFX9-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
3435; GFX9-NEXT:    s_nop 1
3436; GFX9-NEXT:    v_max_i32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
3437; GFX9-NEXT:    s_nop 1
3438; GFX9-NEXT:    v_max_i32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
3439; GFX9-NEXT:    v_readlane_b32 s4, v2, 63
3440; GFX9-NEXT:    s_nop 0
3441; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
3442; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
3443; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
3444; GFX9-NEXT:    ; implicit-def: $vgpr0
3445; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
3446; GFX9-NEXT:    s_cbranch_execz BB17_2
3447; GFX9-NEXT:  ; %bb.1:
3448; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
3449; GFX9-NEXT:    v_mov_b32_e32 v3, s4
3450; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3451; GFX9-NEXT:    ds_max_rtn_i32 v0, v0, v3
3452; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3453; GFX9-NEXT:    buffer_wbinvl1_vol
3454; GFX9-NEXT:  BB17_2:
3455; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
3456; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
3457; GFX9-NEXT:    v_mov_b32_e32 v0, v1
3458; GFX9-NEXT:    v_max_i32_e32 v0, s2, v0
3459; GFX9-NEXT:    s_mov_b32 s3, 0xf000
3460; GFX9-NEXT:    s_mov_b32 s2, -1
3461; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
3462; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3463; GFX9-NEXT:    s_endpgm
3464;
3465; GFX1064-LABEL: max_i32_varying:
3466; GFX1064:       ; %bb.0: ; %entry
3467; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
3468; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
3469; GFX1064-NEXT:    v_bfrev_b32_e32 v1, 1
3470; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
3471; GFX1064-NEXT:    s_not_b64 exec, exec
3472; GFX1064-NEXT:    v_mov_b32_e32 v2, v1
3473; GFX1064-NEXT:    s_not_b64 exec, exec
3474; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
3475; GFX1064-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
3476; GFX1064-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
3477; GFX1064-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
3478; GFX1064-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
3479; GFX1064-NEXT:    v_mov_b32_e32 v3, v2
3480; GFX1064-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
3481; GFX1064-NEXT:    v_max_i32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
3482; GFX1064-NEXT:    v_readlane_b32 s4, v2, 31
3483; GFX1064-NEXT:    v_mov_b32_e32 v3, s4
3484; GFX1064-NEXT:    v_max_i32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
3485; GFX1064-NEXT:    v_readlane_b32 s4, v2, 15
3486; GFX1064-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
3487; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
3488; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
3489; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
3490; GFX1064-NEXT:    v_readlane_b32 s5, v2, 31
3491; GFX1064-NEXT:    v_writelane_b32 v1, s4, 16
3492; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
3493; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
3494; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
3495; GFX1064-NEXT:    v_readlane_b32 s7, v2, 63
3496; GFX1064-NEXT:    v_readlane_b32 s6, v2, 47
3497; GFX1064-NEXT:    v_writelane_b32 v1, s5, 32
3498; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
3499; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
3500; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
3501; GFX1064-NEXT:    v_writelane_b32 v1, s6, 48
3502; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
3503; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
3504; GFX1064-NEXT:    s_mov_b32 s2, -1
3505; GFX1064-NEXT:    ; implicit-def: $vgpr0
3506; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
3507; GFX1064-NEXT:    s_cbranch_execz BB17_2
3508; GFX1064-NEXT:  ; %bb.1:
3509; GFX1064-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
3510; GFX1064-NEXT:    v_mov_b32_e32 v4, s7
3511; GFX1064-NEXT:    s_mov_b32 s3, s7
3512; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3513; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
3514; GFX1064-NEXT:    ds_max_rtn_i32 v0, v7, v4
3515; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3516; GFX1064-NEXT:    buffer_gl0_inv
3517; GFX1064-NEXT:    buffer_gl1_inv
3518; GFX1064-NEXT:  BB17_2:
3519; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
3520; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
3521; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
3522; GFX1064-NEXT:    v_mov_b32_e32 v0, v1
3523; GFX1064-NEXT:    v_max_i32_e32 v0, s3, v0
3524; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
3525; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
3526; GFX1064-NEXT:    s_nop 0
3527; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3528; GFX1064-NEXT:    s_endpgm
3529;
3530; GFX1032-LABEL: max_i32_varying:
3531; GFX1032:       ; %bb.0: ; %entry
3532; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
3533; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
3534; GFX1032-NEXT:    v_bfrev_b32_e32 v1, 1
3535; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
3536; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
3537; GFX1032-NEXT:    v_mov_b32_e32 v2, v1
3538; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
3539; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
3540; GFX1032-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
3541; GFX1032-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
3542; GFX1032-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
3543; GFX1032-NEXT:    v_max_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
3544; GFX1032-NEXT:    v_mov_b32_e32 v3, v2
3545; GFX1032-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
3546; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
3547; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
3548; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
3549; GFX1032-NEXT:    v_max_i32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
3550; GFX1032-NEXT:    v_readlane_b32 s3, v2, 15
3551; GFX1032-NEXT:    v_readlane_b32 s4, v2, 31
3552; GFX1032-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
3553; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
3554; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
3555; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
3556; GFX1032-NEXT:    v_writelane_b32 v1, s3, 16
3557; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
3558; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
3559; GFX1032-NEXT:    s_mov_b32 s2, -1
3560; GFX1032-NEXT:    ; implicit-def: $vgpr0
3561; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
3562; GFX1032-NEXT:    s_cbranch_execz BB17_2
3563; GFX1032-NEXT:  ; %bb.1:
3564; GFX1032-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
3565; GFX1032-NEXT:    v_mov_b32_e32 v4, s4
3566; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3567; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
3568; GFX1032-NEXT:    ds_max_rtn_i32 v0, v7, v4
3569; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3570; GFX1032-NEXT:    buffer_gl0_inv
3571; GFX1032-NEXT:    buffer_gl1_inv
3572; GFX1032-NEXT:  BB17_2:
3573; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
3574; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s3
3575; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
3576; GFX1032-NEXT:    v_mov_b32_e32 v0, v1
3577; GFX1032-NEXT:    v_max_i32_e32 v0, s3, v0
3578; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
3579; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
3580; GFX1032-NEXT:    s_nop 0
3581; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3582; GFX1032-NEXT:    s_endpgm
3583entry:
3584  %lane = call i32 @llvm.amdgcn.workitem.id.x()
3585  %old = atomicrmw max i32 addrspace(3)* @local_var32, i32 %lane acq_rel
3586  store i32 %old, i32 addrspace(1)* %out
3587  ret void
3588}
3589
3590define amdgpu_kernel void @max_i64_constant(i64 addrspace(1)* %out) {
3591;
3592;
3593; GFX7LESS-LABEL: max_i64_constant:
3594; GFX7LESS:       ; %bb.0: ; %entry
3595; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
3596; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
3597; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
3598; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
3599; GFX7LESS-NEXT:    ; implicit-def: $vgpr0_vgpr1
3600; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
3601; GFX7LESS-NEXT:    s_cbranch_execz BB18_2
3602; GFX7LESS-NEXT:  ; %bb.1:
3603; GFX7LESS-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
3604; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 5
3605; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
3606; GFX7LESS-NEXT:    s_mov_b32 m0, -1
3607; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3608; GFX7LESS-NEXT:    ds_max_rtn_i64 v[0:1], v2, v[0:1]
3609; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3610; GFX7LESS-NEXT:    buffer_wbinvl1
3611; GFX7LESS-NEXT:  BB18_2:
3612; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
3613; GFX7LESS-NEXT:    v_readfirstlane_b32 s4, v0
3614; GFX7LESS-NEXT:    v_readfirstlane_b32 s5, v1
3615; GFX7LESS-NEXT:    v_bfrev_b32_e32 v1, 1
3616; GFX7LESS-NEXT:    v_cndmask_b32_e64 v0, 5, 0, vcc
3617; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
3618; GFX7LESS-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
3619; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s5
3620; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s4
3621; GFX7LESS-NEXT:    v_cmp_gt_i64_e32 vcc, s[4:5], v[0:1]
3622; GFX7LESS-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
3623; GFX7LESS-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
3624; GFX7LESS-NEXT:    s_mov_b32 s2, -1
3625; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
3626; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
3627; GFX7LESS-NEXT:    s_endpgm
3628;
3629; GFX8-LABEL: max_i64_constant:
3630; GFX8:       ; %bb.0: ; %entry
3631; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
3632; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
3633; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
3634; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
3635; GFX8-NEXT:    ; implicit-def: $vgpr0_vgpr1
3636; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
3637; GFX8-NEXT:    s_cbranch_execz BB18_2
3638; GFX8-NEXT:  ; %bb.1:
3639; GFX8-NEXT:    v_mov_b32_e32 v0, 5
3640; GFX8-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
3641; GFX8-NEXT:    v_mov_b32_e32 v1, 0
3642; GFX8-NEXT:    s_mov_b32 m0, -1
3643; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3644; GFX8-NEXT:    ds_max_rtn_i64 v[0:1], v2, v[0:1]
3645; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3646; GFX8-NEXT:    buffer_wbinvl1_vol
3647; GFX8-NEXT:  BB18_2:
3648; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
3649; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
3650; GFX8-NEXT:    v_bfrev_b32_e32 v0, 1
3651; GFX8-NEXT:    v_readfirstlane_b32 s3, v1
3652; GFX8-NEXT:    v_cndmask_b32_e32 v1, 0, v0, vcc
3653; GFX8-NEXT:    v_cndmask_b32_e64 v0, 5, 0, vcc
3654; GFX8-NEXT:    v_cmp_gt_i64_e32 vcc, s[2:3], v[0:1]
3655; GFX8-NEXT:    v_mov_b32_e32 v2, s3
3656; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
3657; GFX8-NEXT:    v_mov_b32_e32 v2, s2
3658; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
3659; GFX8-NEXT:    s_mov_b32 s3, 0xf000
3660; GFX8-NEXT:    s_mov_b32 s2, -1
3661; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
3662; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
3663; GFX8-NEXT:    s_endpgm
3664;
3665; GFX9-LABEL: max_i64_constant:
3666; GFX9:       ; %bb.0: ; %entry
3667; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
3668; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
3669; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
3670; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
3671; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1
3672; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
3673; GFX9-NEXT:    s_cbranch_execz BB18_2
3674; GFX9-NEXT:  ; %bb.1:
3675; GFX9-NEXT:    v_mov_b32_e32 v0, 5
3676; GFX9-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
3677; GFX9-NEXT:    v_mov_b32_e32 v1, 0
3678; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3679; GFX9-NEXT:    ds_max_rtn_i64 v[0:1], v2, v[0:1]
3680; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3681; GFX9-NEXT:    buffer_wbinvl1_vol
3682; GFX9-NEXT:  BB18_2:
3683; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
3684; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
3685; GFX9-NEXT:    v_bfrev_b32_e32 v0, 1
3686; GFX9-NEXT:    v_readfirstlane_b32 s3, v1
3687; GFX9-NEXT:    v_cndmask_b32_e32 v1, 0, v0, vcc
3688; GFX9-NEXT:    v_cndmask_b32_e64 v0, 5, 0, vcc
3689; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, s[2:3], v[0:1]
3690; GFX9-NEXT:    v_mov_b32_e32 v2, s3
3691; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
3692; GFX9-NEXT:    v_mov_b32_e32 v2, s2
3693; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
3694; GFX9-NEXT:    s_mov_b32 s3, 0xf000
3695; GFX9-NEXT:    s_mov_b32 s2, -1
3696; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
3697; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
3698; GFX9-NEXT:    s_endpgm
3699;
3700; GFX1064-LABEL: max_i64_constant:
3701; GFX1064:       ; %bb.0: ; %entry
3702; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
3703; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
3704; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
3705; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
3706; GFX1064-NEXT:    ; implicit-def: $vgpr0_vgpr1
3707; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
3708; GFX1064-NEXT:    s_cbranch_execz BB18_2
3709; GFX1064-NEXT:  ; %bb.1:
3710; GFX1064-NEXT:    v_mov_b32_e32 v0, 5
3711; GFX1064-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
3712; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
3713; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3714; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
3715; GFX1064-NEXT:    ds_max_rtn_i64 v[0:1], v2, v[0:1]
3716; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3717; GFX1064-NEXT:    buffer_gl0_inv
3718; GFX1064-NEXT:    buffer_gl1_inv
3719; GFX1064-NEXT:  BB18_2:
3720; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
3721; GFX1064-NEXT:    s_or_b64 exec, exec, s[2:3]
3722; GFX1064-NEXT:    v_readfirstlane_b32 s2, v0
3723; GFX1064-NEXT:    v_readfirstlane_b32 s3, v1
3724; GFX1064-NEXT:    v_cndmask_b32_e64 v1, 0, 0x80000000, vcc
3725; GFX1064-NEXT:    v_cndmask_b32_e64 v0, 5, 0, vcc
3726; GFX1064-NEXT:    v_cmp_gt_i64_e32 vcc, s[2:3], v[0:1]
3727; GFX1064-NEXT:    v_cndmask_b32_e64 v1, v1, s3, vcc
3728; GFX1064-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc
3729; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
3730; GFX1064-NEXT:    s_mov_b32 s2, -1
3731; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
3732; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
3733; GFX1064-NEXT:    s_endpgm
3734;
3735; GFX1032-LABEL: max_i64_constant:
3736; GFX1032:       ; %bb.0: ; %entry
3737; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
3738; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
3739; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
3740; GFX1032-NEXT:    ; implicit-def: $vgpr0_vgpr1
3741; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
3742; GFX1032-NEXT:    s_cbranch_execz BB18_2
3743; GFX1032-NEXT:  ; %bb.1:
3744; GFX1032-NEXT:    v_mov_b32_e32 v0, 5
3745; GFX1032-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
3746; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
3747; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3748; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
3749; GFX1032-NEXT:    ds_max_rtn_i64 v[0:1], v2, v[0:1]
3750; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3751; GFX1032-NEXT:    buffer_gl0_inv
3752; GFX1032-NEXT:    buffer_gl1_inv
3753; GFX1032-NEXT:  BB18_2:
3754; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
3755; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2
3756; GFX1032-NEXT:    v_readfirstlane_b32 s2, v0
3757; GFX1032-NEXT:    v_readfirstlane_b32 s3, v1
3758; GFX1032-NEXT:    v_cndmask_b32_e64 v1, 0, 0x80000000, vcc_lo
3759; GFX1032-NEXT:    v_cndmask_b32_e64 v0, 5, 0, vcc_lo
3760; GFX1032-NEXT:    v_cmp_gt_i64_e32 vcc_lo, s[2:3], v[0:1]
3761; GFX1032-NEXT:    v_cndmask_b32_e64 v1, v1, s3, vcc_lo
3762; GFX1032-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
3763; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
3764; GFX1032-NEXT:    s_mov_b32 s2, -1
3765; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
3766; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
3767; GFX1032-NEXT:    s_endpgm
3768entry:
3769  %old = atomicrmw max i64 addrspace(3)* @local_var64, i64 5 acq_rel
3770  store i64 %old, i64 addrspace(1)* %out
3771  ret void
3772}
3773
3774define amdgpu_kernel void @min_i32_varying(i32 addrspace(1)* %out) {
3775;
3776;
3777; GFX7LESS-LABEL: min_i32_varying:
3778; GFX7LESS:       ; %bb.0: ; %entry
3779; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
3780; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
3781; GFX7LESS-NEXT:    s_mov_b32 m0, -1
3782; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3783; GFX7LESS-NEXT:    ds_min_rtn_i32 v0, v1, v0
3784; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3785; GFX7LESS-NEXT:    buffer_wbinvl1
3786; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
3787; GFX7LESS-NEXT:    s_mov_b32 s2, -1
3788; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3789; GFX7LESS-NEXT:    s_endpgm
3790;
3791; GFX8-LABEL: min_i32_varying:
3792; GFX8:       ; %bb.0: ; %entry
3793; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
3794; GFX8-NEXT:    v_mov_b32_e32 v2, v0
3795; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
3796; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
3797; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
3798; GFX8-NEXT:    v_bfrev_b32_e32 v1, -2
3799; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
3800; GFX8-NEXT:    s_not_b64 exec, exec
3801; GFX8-NEXT:    v_mov_b32_e32 v2, v1
3802; GFX8-NEXT:    s_not_b64 exec, exec
3803; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
3804; GFX8-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
3805; GFX8-NEXT:    s_nop 1
3806; GFX8-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
3807; GFX8-NEXT:    s_nop 1
3808; GFX8-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
3809; GFX8-NEXT:    s_nop 1
3810; GFX8-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
3811; GFX8-NEXT:    s_nop 1
3812; GFX8-NEXT:    v_min_i32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
3813; GFX8-NEXT:    s_nop 1
3814; GFX8-NEXT:    v_min_i32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
3815; GFX8-NEXT:    v_readlane_b32 s4, v2, 63
3816; GFX8-NEXT:    s_nop 0
3817; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
3818; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
3819; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
3820; GFX8-NEXT:    ; implicit-def: $vgpr0
3821; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
3822; GFX8-NEXT:    s_cbranch_execz BB19_2
3823; GFX8-NEXT:  ; %bb.1:
3824; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
3825; GFX8-NEXT:    v_mov_b32_e32 v3, s4
3826; GFX8-NEXT:    s_mov_b32 m0, -1
3827; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3828; GFX8-NEXT:    ds_min_rtn_i32 v0, v0, v3
3829; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3830; GFX8-NEXT:    buffer_wbinvl1_vol
3831; GFX8-NEXT:  BB19_2:
3832; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
3833; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
3834; GFX8-NEXT:    v_mov_b32_e32 v0, v1
3835; GFX8-NEXT:    v_min_i32_e32 v0, s2, v0
3836; GFX8-NEXT:    s_mov_b32 s3, 0xf000
3837; GFX8-NEXT:    s_mov_b32 s2, -1
3838; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
3839; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3840; GFX8-NEXT:    s_endpgm
3841;
3842; GFX9-LABEL: min_i32_varying:
3843; GFX9:       ; %bb.0: ; %entry
3844; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
3845; GFX9-NEXT:    v_mov_b32_e32 v2, v0
3846; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
3847; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
3848; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
3849; GFX9-NEXT:    v_bfrev_b32_e32 v1, -2
3850; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
3851; GFX9-NEXT:    s_not_b64 exec, exec
3852; GFX9-NEXT:    v_mov_b32_e32 v2, v1
3853; GFX9-NEXT:    s_not_b64 exec, exec
3854; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
3855; GFX9-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
3856; GFX9-NEXT:    s_nop 1
3857; GFX9-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
3858; GFX9-NEXT:    s_nop 1
3859; GFX9-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
3860; GFX9-NEXT:    s_nop 1
3861; GFX9-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
3862; GFX9-NEXT:    s_nop 1
3863; GFX9-NEXT:    v_min_i32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
3864; GFX9-NEXT:    s_nop 1
3865; GFX9-NEXT:    v_min_i32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
3866; GFX9-NEXT:    v_readlane_b32 s4, v2, 63
3867; GFX9-NEXT:    s_nop 0
3868; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
3869; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
3870; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
3871; GFX9-NEXT:    ; implicit-def: $vgpr0
3872; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
3873; GFX9-NEXT:    s_cbranch_execz BB19_2
3874; GFX9-NEXT:  ; %bb.1:
3875; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
3876; GFX9-NEXT:    v_mov_b32_e32 v3, s4
3877; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3878; GFX9-NEXT:    ds_min_rtn_i32 v0, v0, v3
3879; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3880; GFX9-NEXT:    buffer_wbinvl1_vol
3881; GFX9-NEXT:  BB19_2:
3882; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
3883; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
3884; GFX9-NEXT:    v_mov_b32_e32 v0, v1
3885; GFX9-NEXT:    v_min_i32_e32 v0, s2, v0
3886; GFX9-NEXT:    s_mov_b32 s3, 0xf000
3887; GFX9-NEXT:    s_mov_b32 s2, -1
3888; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
3889; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3890; GFX9-NEXT:    s_endpgm
3891;
3892; GFX1064-LABEL: min_i32_varying:
3893; GFX1064:       ; %bb.0: ; %entry
3894; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
3895; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
3896; GFX1064-NEXT:    v_bfrev_b32_e32 v1, -2
3897; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
3898; GFX1064-NEXT:    s_not_b64 exec, exec
3899; GFX1064-NEXT:    v_mov_b32_e32 v2, v1
3900; GFX1064-NEXT:    s_not_b64 exec, exec
3901; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
3902; GFX1064-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
3903; GFX1064-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
3904; GFX1064-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
3905; GFX1064-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
3906; GFX1064-NEXT:    v_mov_b32_e32 v3, v2
3907; GFX1064-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
3908; GFX1064-NEXT:    v_min_i32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
3909; GFX1064-NEXT:    v_readlane_b32 s4, v2, 31
3910; GFX1064-NEXT:    v_mov_b32_e32 v3, s4
3911; GFX1064-NEXT:    v_min_i32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
3912; GFX1064-NEXT:    v_readlane_b32 s4, v2, 15
3913; GFX1064-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
3914; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
3915; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
3916; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
3917; GFX1064-NEXT:    v_readlane_b32 s5, v2, 31
3918; GFX1064-NEXT:    v_writelane_b32 v1, s4, 16
3919; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
3920; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
3921; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
3922; GFX1064-NEXT:    v_readlane_b32 s7, v2, 63
3923; GFX1064-NEXT:    v_readlane_b32 s6, v2, 47
3924; GFX1064-NEXT:    v_writelane_b32 v1, s5, 32
3925; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
3926; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
3927; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
3928; GFX1064-NEXT:    v_writelane_b32 v1, s6, 48
3929; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
3930; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
3931; GFX1064-NEXT:    s_mov_b32 s2, -1
3932; GFX1064-NEXT:    ; implicit-def: $vgpr0
3933; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
3934; GFX1064-NEXT:    s_cbranch_execz BB19_2
3935; GFX1064-NEXT:  ; %bb.1:
3936; GFX1064-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
3937; GFX1064-NEXT:    v_mov_b32_e32 v4, s7
3938; GFX1064-NEXT:    s_mov_b32 s3, s7
3939; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3940; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
3941; GFX1064-NEXT:    ds_min_rtn_i32 v0, v7, v4
3942; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3943; GFX1064-NEXT:    buffer_gl0_inv
3944; GFX1064-NEXT:    buffer_gl1_inv
3945; GFX1064-NEXT:  BB19_2:
3946; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
3947; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
3948; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
3949; GFX1064-NEXT:    v_mov_b32_e32 v0, v1
3950; GFX1064-NEXT:    v_min_i32_e32 v0, s3, v0
3951; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
3952; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
3953; GFX1064-NEXT:    s_nop 0
3954; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3955; GFX1064-NEXT:    s_endpgm
3956;
3957; GFX1032-LABEL: min_i32_varying:
3958; GFX1032:       ; %bb.0: ; %entry
3959; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
3960; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
3961; GFX1032-NEXT:    v_bfrev_b32_e32 v1, -2
3962; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
3963; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
3964; GFX1032-NEXT:    v_mov_b32_e32 v2, v1
3965; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
3966; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
3967; GFX1032-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
3968; GFX1032-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
3969; GFX1032-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
3970; GFX1032-NEXT:    v_min_i32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
3971; GFX1032-NEXT:    v_mov_b32_e32 v3, v2
3972; GFX1032-NEXT:    v_permlanex16_b32 v3, v3, -1, -1
3973; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
3974; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
3975; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
3976; GFX1032-NEXT:    v_min_i32_dpp v2, v3, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
3977; GFX1032-NEXT:    v_readlane_b32 s3, v2, 15
3978; GFX1032-NEXT:    v_readlane_b32 s4, v2, 31
3979; GFX1032-NEXT:    v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf
3980; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
3981; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
3982; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
3983; GFX1032-NEXT:    v_writelane_b32 v1, s3, 16
3984; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
3985; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
3986; GFX1032-NEXT:    s_mov_b32 s2, -1
3987; GFX1032-NEXT:    ; implicit-def: $vgpr0
3988; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
3989; GFX1032-NEXT:    s_cbranch_execz BB19_2
3990; GFX1032-NEXT:  ; %bb.1:
3991; GFX1032-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
3992; GFX1032-NEXT:    v_mov_b32_e32 v4, s4
3993; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3994; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
3995; GFX1032-NEXT:    ds_min_rtn_i32 v0, v7, v4
3996; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3997; GFX1032-NEXT:    buffer_gl0_inv
3998; GFX1032-NEXT:    buffer_gl1_inv
3999; GFX1032-NEXT:  BB19_2:
4000; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
4001; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s3
4002; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
4003; GFX1032-NEXT:    v_mov_b32_e32 v0, v1
4004; GFX1032-NEXT:    v_min_i32_e32 v0, s3, v0
4005; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
4006; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
4007; GFX1032-NEXT:    s_nop 0
4008; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4009; GFX1032-NEXT:    s_endpgm
4010entry:
4011  %lane = call i32 @llvm.amdgcn.workitem.id.x()
4012  %old = atomicrmw min i32 addrspace(3)* @local_var32, i32 %lane acq_rel
4013  store i32 %old, i32 addrspace(1)* %out
4014  ret void
4015}
4016
4017define amdgpu_kernel void @min_i64_constant(i64 addrspace(1)* %out) {
4018;
4019;
4020; GFX7LESS-LABEL: min_i64_constant:
4021; GFX7LESS:       ; %bb.0: ; %entry
4022; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
4023; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
4024; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
4025; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
4026; GFX7LESS-NEXT:    ; implicit-def: $vgpr0_vgpr1
4027; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
4028; GFX7LESS-NEXT:    s_cbranch_execz BB20_2
4029; GFX7LESS-NEXT:  ; %bb.1:
4030; GFX7LESS-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
4031; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 5
4032; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
4033; GFX7LESS-NEXT:    s_mov_b32 m0, -1
4034; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4035; GFX7LESS-NEXT:    ds_min_rtn_i64 v[0:1], v2, v[0:1]
4036; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4037; GFX7LESS-NEXT:    buffer_wbinvl1
4038; GFX7LESS-NEXT:  BB20_2:
4039; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
4040; GFX7LESS-NEXT:    v_readfirstlane_b32 s4, v0
4041; GFX7LESS-NEXT:    v_readfirstlane_b32 s5, v1
4042; GFX7LESS-NEXT:    v_bfrev_b32_e32 v1, -2
4043; GFX7LESS-NEXT:    s_mov_b32 s2, -1
4044; GFX7LESS-NEXT:    v_cndmask_b32_e64 v0, 5, -1, vcc
4045; GFX7LESS-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
4046; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s5
4047; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s4
4048; GFX7LESS-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
4049; GFX7LESS-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
4050; GFX7LESS-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
4051; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
4052; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
4053; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
4054; GFX7LESS-NEXT:    s_endpgm
4055;
4056; GFX8-LABEL: min_i64_constant:
4057; GFX8:       ; %bb.0: ; %entry
4058; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
4059; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
4060; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
4061; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
4062; GFX8-NEXT:    ; implicit-def: $vgpr0_vgpr1
4063; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
4064; GFX8-NEXT:    s_cbranch_execz BB20_2
4065; GFX8-NEXT:  ; %bb.1:
4066; GFX8-NEXT:    v_mov_b32_e32 v0, 5
4067; GFX8-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
4068; GFX8-NEXT:    v_mov_b32_e32 v1, 0
4069; GFX8-NEXT:    s_mov_b32 m0, -1
4070; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4071; GFX8-NEXT:    ds_min_rtn_i64 v[0:1], v2, v[0:1]
4072; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4073; GFX8-NEXT:    buffer_wbinvl1_vol
4074; GFX8-NEXT:  BB20_2:
4075; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
4076; GFX8-NEXT:    v_readfirstlane_b32 s4, v0
4077; GFX8-NEXT:    v_bfrev_b32_e32 v0, -2
4078; GFX8-NEXT:    v_readfirstlane_b32 s5, v1
4079; GFX8-NEXT:    v_cndmask_b32_e32 v1, 0, v0, vcc
4080; GFX8-NEXT:    v_cndmask_b32_e64 v0, 5, -1, vcc
4081; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
4082; GFX8-NEXT:    v_mov_b32_e32 v2, s5
4083; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
4084; GFX8-NEXT:    v_mov_b32_e32 v2, s4
4085; GFX8-NEXT:    s_mov_b32 s2, -1
4086; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
4087; GFX8-NEXT:    s_mov_b32 s3, 0xf000
4088; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
4089; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
4090; GFX8-NEXT:    s_endpgm
4091;
4092; GFX9-LABEL: min_i64_constant:
4093; GFX9:       ; %bb.0: ; %entry
4094; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
4095; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
4096; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
4097; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
4098; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1
4099; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
4100; GFX9-NEXT:    s_cbranch_execz BB20_2
4101; GFX9-NEXT:  ; %bb.1:
4102; GFX9-NEXT:    v_mov_b32_e32 v0, 5
4103; GFX9-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
4104; GFX9-NEXT:    v_mov_b32_e32 v1, 0
4105; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4106; GFX9-NEXT:    ds_min_rtn_i64 v[0:1], v2, v[0:1]
4107; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4108; GFX9-NEXT:    buffer_wbinvl1_vol
4109; GFX9-NEXT:  BB20_2:
4110; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
4111; GFX9-NEXT:    v_readfirstlane_b32 s4, v0
4112; GFX9-NEXT:    v_bfrev_b32_e32 v0, -2
4113; GFX9-NEXT:    v_readfirstlane_b32 s5, v1
4114; GFX9-NEXT:    v_cndmask_b32_e32 v1, 0, v0, vcc
4115; GFX9-NEXT:    v_cndmask_b32_e64 v0, 5, -1, vcc
4116; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
4117; GFX9-NEXT:    v_mov_b32_e32 v2, s5
4118; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
4119; GFX9-NEXT:    v_mov_b32_e32 v2, s4
4120; GFX9-NEXT:    s_mov_b32 s2, -1
4121; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
4122; GFX9-NEXT:    s_mov_b32 s3, 0xf000
4123; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
4124; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
4125; GFX9-NEXT:    s_endpgm
4126;
4127; GFX1064-LABEL: min_i64_constant:
4128; GFX1064:       ; %bb.0: ; %entry
4129; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
4130; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
4131; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
4132; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
4133; GFX1064-NEXT:    ; implicit-def: $vgpr0_vgpr1
4134; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
4135; GFX1064-NEXT:    s_cbranch_execz BB20_2
4136; GFX1064-NEXT:  ; %bb.1:
4137; GFX1064-NEXT:    v_mov_b32_e32 v0, 5
4138; GFX1064-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
4139; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
4140; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4141; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
4142; GFX1064-NEXT:    ds_min_rtn_i64 v[0:1], v2, v[0:1]
4143; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4144; GFX1064-NEXT:    buffer_gl0_inv
4145; GFX1064-NEXT:    buffer_gl1_inv
4146; GFX1064-NEXT:  BB20_2:
4147; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
4148; GFX1064-NEXT:    s_or_b64 exec, exec, s[2:3]
4149; GFX1064-NEXT:    v_readfirstlane_b32 s2, v0
4150; GFX1064-NEXT:    v_readfirstlane_b32 s3, v1
4151; GFX1064-NEXT:    v_cndmask_b32_e64 v1, 0, 0x7fffffff, vcc
4152; GFX1064-NEXT:    v_cndmask_b32_e64 v0, 5, -1, vcc
4153; GFX1064-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
4154; GFX1064-NEXT:    v_cndmask_b32_e64 v1, v1, s3, vcc
4155; GFX1064-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc
4156; GFX1064-NEXT:    s_mov_b32 s2, -1
4157; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
4158; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
4159; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
4160; GFX1064-NEXT:    s_endpgm
4161;
4162; GFX1032-LABEL: min_i64_constant:
4163; GFX1032:       ; %bb.0: ; %entry
4164; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
4165; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
4166; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
4167; GFX1032-NEXT:    ; implicit-def: $vgpr0_vgpr1
4168; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
4169; GFX1032-NEXT:    s_cbranch_execz BB20_2
4170; GFX1032-NEXT:  ; %bb.1:
4171; GFX1032-NEXT:    v_mov_b32_e32 v0, 5
4172; GFX1032-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
4173; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
4174; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4175; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
4176; GFX1032-NEXT:    ds_min_rtn_i64 v[0:1], v2, v[0:1]
4177; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4178; GFX1032-NEXT:    buffer_gl0_inv
4179; GFX1032-NEXT:    buffer_gl1_inv
4180; GFX1032-NEXT:  BB20_2:
4181; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
4182; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2
4183; GFX1032-NEXT:    v_readfirstlane_b32 s2, v0
4184; GFX1032-NEXT:    v_readfirstlane_b32 s3, v1
4185; GFX1032-NEXT:    v_cndmask_b32_e64 v1, 0, 0x7fffffff, vcc_lo
4186; GFX1032-NEXT:    v_cndmask_b32_e64 v0, 5, -1, vcc_lo
4187; GFX1032-NEXT:    v_cmp_lt_i64_e32 vcc_lo, s[2:3], v[0:1]
4188; GFX1032-NEXT:    v_cndmask_b32_e64 v1, v1, s3, vcc_lo
4189; GFX1032-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
4190; GFX1032-NEXT:    s_mov_b32 s2, -1
4191; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
4192; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
4193; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
4194; GFX1032-NEXT:    s_endpgm
4195entry:
4196  %old = atomicrmw min i64 addrspace(3)* @local_var64, i64 5 acq_rel
4197  store i64 %old, i64 addrspace(1)* %out
4198  ret void
4199}
4200
4201define amdgpu_kernel void @umax_i32_varying(i32 addrspace(1)* %out) {
4202;
4203;
4204; GFX7LESS-LABEL: umax_i32_varying:
4205; GFX7LESS:       ; %bb.0: ; %entry
4206; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
4207; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
4208; GFX7LESS-NEXT:    s_mov_b32 m0, -1
4209; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4210; GFX7LESS-NEXT:    ds_max_rtn_u32 v0, v1, v0
4211; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4212; GFX7LESS-NEXT:    buffer_wbinvl1
4213; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
4214; GFX7LESS-NEXT:    s_mov_b32 s2, -1
4215; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4216; GFX7LESS-NEXT:    s_endpgm
4217;
4218; GFX8-LABEL: umax_i32_varying:
4219; GFX8:       ; %bb.0: ; %entry
4220; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
4221; GFX8-NEXT:    v_mov_b32_e32 v2, v0
4222; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
4223; GFX8-NEXT:    v_mov_b32_e32 v1, 0
4224; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
4225; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
4226; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
4227; GFX8-NEXT:    s_not_b64 exec, exec
4228; GFX8-NEXT:    v_mov_b32_e32 v2, 0
4229; GFX8-NEXT:    s_not_b64 exec, exec
4230; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
4231; GFX8-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
4232; GFX8-NEXT:    s_nop 1
4233; GFX8-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
4234; GFX8-NEXT:    s_nop 1
4235; GFX8-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
4236; GFX8-NEXT:    s_nop 1
4237; GFX8-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
4238; GFX8-NEXT:    s_nop 1
4239; GFX8-NEXT:    v_max_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
4240; GFX8-NEXT:    s_nop 1
4241; GFX8-NEXT:    v_max_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
4242; GFX8-NEXT:    v_readlane_b32 s4, v2, 63
4243; GFX8-NEXT:    s_nop 0
4244; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
4245; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
4246; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
4247; GFX8-NEXT:    ; implicit-def: $vgpr0
4248; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
4249; GFX8-NEXT:    s_cbranch_execz BB21_2
4250; GFX8-NEXT:  ; %bb.1:
4251; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
4252; GFX8-NEXT:    v_mov_b32_e32 v3, s4
4253; GFX8-NEXT:    s_mov_b32 m0, -1
4254; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4255; GFX8-NEXT:    ds_max_rtn_u32 v0, v0, v3
4256; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4257; GFX8-NEXT:    buffer_wbinvl1_vol
4258; GFX8-NEXT:  BB21_2:
4259; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
4260; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
4261; GFX8-NEXT:    v_mov_b32_e32 v0, v1
4262; GFX8-NEXT:    v_max_u32_e32 v0, s2, v0
4263; GFX8-NEXT:    s_mov_b32 s3, 0xf000
4264; GFX8-NEXT:    s_mov_b32 s2, -1
4265; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
4266; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4267; GFX8-NEXT:    s_endpgm
4268;
4269; GFX9-LABEL: umax_i32_varying:
4270; GFX9:       ; %bb.0: ; %entry
4271; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
4272; GFX9-NEXT:    v_mov_b32_e32 v2, v0
4273; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
4274; GFX9-NEXT:    v_mov_b32_e32 v1, 0
4275; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
4276; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
4277; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
4278; GFX9-NEXT:    s_not_b64 exec, exec
4279; GFX9-NEXT:    v_mov_b32_e32 v2, 0
4280; GFX9-NEXT:    s_not_b64 exec, exec
4281; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
4282; GFX9-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
4283; GFX9-NEXT:    s_nop 1
4284; GFX9-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
4285; GFX9-NEXT:    s_nop 1
4286; GFX9-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
4287; GFX9-NEXT:    s_nop 1
4288; GFX9-NEXT:    v_max_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
4289; GFX9-NEXT:    s_nop 1
4290; GFX9-NEXT:    v_max_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
4291; GFX9-NEXT:    s_nop 1
4292; GFX9-NEXT:    v_max_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
4293; GFX9-NEXT:    v_readlane_b32 s4, v2, 63
4294; GFX9-NEXT:    s_nop 0
4295; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
4296; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
4297; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
4298; GFX9-NEXT:    ; implicit-def: $vgpr0
4299; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
4300; GFX9-NEXT:    s_cbranch_execz BB21_2
4301; GFX9-NEXT:  ; %bb.1:
4302; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
4303; GFX9-NEXT:    v_mov_b32_e32 v3, s4
4304; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4305; GFX9-NEXT:    ds_max_rtn_u32 v0, v0, v3
4306; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4307; GFX9-NEXT:    buffer_wbinvl1_vol
4308; GFX9-NEXT:  BB21_2:
4309; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
4310; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
4311; GFX9-NEXT:    v_mov_b32_e32 v0, v1
4312; GFX9-NEXT:    v_max_u32_e32 v0, s2, v0
4313; GFX9-NEXT:    s_mov_b32 s3, 0xf000
4314; GFX9-NEXT:    s_mov_b32 s2, -1
4315; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
4316; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4317; GFX9-NEXT:    s_endpgm
4318;
4319; GFX1064-LABEL: umax_i32_varying:
4320; GFX1064:       ; %bb.0: ; %entry
4321; GFX1064-NEXT:    v_mov_b32_e32 v1, v0
4322; GFX1064-NEXT:    s_not_b64 exec, exec
4323; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
4324; GFX1064-NEXT:    s_not_b64 exec, exec
4325; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
4326; GFX1064-NEXT:    v_max_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
4327; GFX1064-NEXT:    v_mov_b32_e32 v3, 0
4328; GFX1064-NEXT:    v_max_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
4329; GFX1064-NEXT:    v_max_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
4330; GFX1064-NEXT:    v_max_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
4331; GFX1064-NEXT:    v_mov_b32_e32 v2, v1
4332; GFX1064-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
4333; GFX1064-NEXT:    v_max_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
4334; GFX1064-NEXT:    v_readlane_b32 s4, v1, 31
4335; GFX1064-NEXT:    v_mov_b32_e32 v2, s4
4336; GFX1064-NEXT:    v_max_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
4337; GFX1064-NEXT:    v_readlane_b32 s4, v1, 15
4338; GFX1064-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
4339; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
4340; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
4341; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
4342; GFX1064-NEXT:    v_readlane_b32 s5, v1, 31
4343; GFX1064-NEXT:    v_writelane_b32 v3, s4, 16
4344; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
4345; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
4346; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
4347; GFX1064-NEXT:    v_readlane_b32 s7, v1, 63
4348; GFX1064-NEXT:    v_readlane_b32 s6, v1, 47
4349; GFX1064-NEXT:    v_writelane_b32 v3, s5, 32
4350; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
4351; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
4352; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
4353; GFX1064-NEXT:    v_writelane_b32 v3, s6, 48
4354; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
4355; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
4356; GFX1064-NEXT:    s_mov_b32 s2, -1
4357; GFX1064-NEXT:    ; implicit-def: $vgpr0
4358; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
4359; GFX1064-NEXT:    s_cbranch_execz BB21_2
4360; GFX1064-NEXT:  ; %bb.1:
4361; GFX1064-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
4362; GFX1064-NEXT:    v_mov_b32_e32 v4, s7
4363; GFX1064-NEXT:    s_mov_b32 s3, s7
4364; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4365; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
4366; GFX1064-NEXT:    ds_max_rtn_u32 v0, v7, v4
4367; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4368; GFX1064-NEXT:    buffer_gl0_inv
4369; GFX1064-NEXT:    buffer_gl1_inv
4370; GFX1064-NEXT:  BB21_2:
4371; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
4372; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
4373; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
4374; GFX1064-NEXT:    v_mov_b32_e32 v0, v3
4375; GFX1064-NEXT:    v_max_u32_e32 v0, s3, v0
4376; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
4377; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
4378; GFX1064-NEXT:    s_nop 0
4379; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4380; GFX1064-NEXT:    s_endpgm
4381;
4382; GFX1032-LABEL: umax_i32_varying:
4383; GFX1032:       ; %bb.0: ; %entry
4384; GFX1032-NEXT:    v_mov_b32_e32 v1, v0
4385; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
4386; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
4387; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
4388; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
4389; GFX1032-NEXT:    v_max_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:0
4390; GFX1032-NEXT:    v_max_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:0
4391; GFX1032-NEXT:    v_max_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:0
4392; GFX1032-NEXT:    v_max_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:0
4393; GFX1032-NEXT:    v_mov_b32_e32 v2, v1
4394; GFX1032-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
4395; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
4396; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
4397; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
4398; GFX1032-NEXT:    v_max_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
4399; GFX1032-NEXT:    v_mov_b32_e32 v3, 0
4400; GFX1032-NEXT:    v_readlane_b32 s3, v1, 15
4401; GFX1032-NEXT:    v_readlane_b32 s4, v1, 31
4402; GFX1032-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
4403; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
4404; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
4405; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
4406; GFX1032-NEXT:    v_writelane_b32 v3, s3, 16
4407; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
4408; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
4409; GFX1032-NEXT:    s_mov_b32 s2, -1
4410; GFX1032-NEXT:    ; implicit-def: $vgpr0
4411; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
4412; GFX1032-NEXT:    s_cbranch_execz BB21_2
4413; GFX1032-NEXT:  ; %bb.1:
4414; GFX1032-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
4415; GFX1032-NEXT:    v_mov_b32_e32 v4, s4
4416; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4417; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
4418; GFX1032-NEXT:    ds_max_rtn_u32 v0, v7, v4
4419; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4420; GFX1032-NEXT:    buffer_gl0_inv
4421; GFX1032-NEXT:    buffer_gl1_inv
4422; GFX1032-NEXT:  BB21_2:
4423; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
4424; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s3
4425; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
4426; GFX1032-NEXT:    v_mov_b32_e32 v0, v3
4427; GFX1032-NEXT:    v_max_u32_e32 v0, s3, v0
4428; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
4429; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
4430; GFX1032-NEXT:    s_nop 0
4431; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4432; GFX1032-NEXT:    s_endpgm
4433entry:
4434  %lane = call i32 @llvm.amdgcn.workitem.id.x()
4435  %old = atomicrmw umax i32 addrspace(3)* @local_var32, i32 %lane acq_rel
4436  store i32 %old, i32 addrspace(1)* %out
4437  ret void
4438}
4439
4440define amdgpu_kernel void @umax_i64_constant(i64 addrspace(1)* %out) {
4441;
4442;
4443; GFX7LESS-LABEL: umax_i64_constant:
4444; GFX7LESS:       ; %bb.0: ; %entry
4445; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
4446; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
4447; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
4448; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
4449; GFX7LESS-NEXT:    ; implicit-def: $vgpr0_vgpr1
4450; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
4451; GFX7LESS-NEXT:    s_cbranch_execz BB22_2
4452; GFX7LESS-NEXT:  ; %bb.1:
4453; GFX7LESS-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
4454; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 5
4455; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
4456; GFX7LESS-NEXT:    s_mov_b32 m0, -1
4457; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4458; GFX7LESS-NEXT:    ds_max_rtn_u64 v[0:1], v2, v[0:1]
4459; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4460; GFX7LESS-NEXT:    buffer_wbinvl1
4461; GFX7LESS-NEXT:  BB22_2:
4462; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
4463; GFX7LESS-NEXT:    v_readfirstlane_b32 s4, v0
4464; GFX7LESS-NEXT:    v_readfirstlane_b32 s5, v1
4465; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
4466; GFX7LESS-NEXT:    v_cndmask_b32_e64 v0, 5, 0, vcc
4467; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
4468; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s4
4469; GFX7LESS-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[0:1]
4470; GFX7LESS-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
4471; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s5
4472; GFX7LESS-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
4473; GFX7LESS-NEXT:    s_mov_b32 s2, -1
4474; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
4475; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
4476; GFX7LESS-NEXT:    s_endpgm
4477;
4478; GFX8-LABEL: umax_i64_constant:
4479; GFX8:       ; %bb.0: ; %entry
4480; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
4481; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
4482; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
4483; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
4484; GFX8-NEXT:    ; implicit-def: $vgpr0_vgpr1
4485; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
4486; GFX8-NEXT:    s_cbranch_execz BB22_2
4487; GFX8-NEXT:  ; %bb.1:
4488; GFX8-NEXT:    v_mov_b32_e32 v0, 5
4489; GFX8-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
4490; GFX8-NEXT:    v_mov_b32_e32 v1, 0
4491; GFX8-NEXT:    s_mov_b32 m0, -1
4492; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4493; GFX8-NEXT:    ds_max_rtn_u64 v[0:1], v2, v[0:1]
4494; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4495; GFX8-NEXT:    buffer_wbinvl1_vol
4496; GFX8-NEXT:  BB22_2:
4497; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
4498; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
4499; GFX8-NEXT:    v_readfirstlane_b32 s3, v1
4500; GFX8-NEXT:    v_mov_b32_e32 v1, 0
4501; GFX8-NEXT:    v_cndmask_b32_e64 v0, 5, 0, vcc
4502; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
4503; GFX8-NEXT:    v_mov_b32_e32 v1, s3
4504; GFX8-NEXT:    v_mov_b32_e32 v2, s2
4505; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
4506; GFX8-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
4507; GFX8-NEXT:    s_mov_b32 s3, 0xf000
4508; GFX8-NEXT:    s_mov_b32 s2, -1
4509; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
4510; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
4511; GFX8-NEXT:    s_endpgm
4512;
4513; GFX9-LABEL: umax_i64_constant:
4514; GFX9:       ; %bb.0: ; %entry
4515; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
4516; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
4517; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
4518; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
4519; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1
4520; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
4521; GFX9-NEXT:    s_cbranch_execz BB22_2
4522; GFX9-NEXT:  ; %bb.1:
4523; GFX9-NEXT:    v_mov_b32_e32 v0, 5
4524; GFX9-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
4525; GFX9-NEXT:    v_mov_b32_e32 v1, 0
4526; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4527; GFX9-NEXT:    ds_max_rtn_u64 v[0:1], v2, v[0:1]
4528; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4529; GFX9-NEXT:    buffer_wbinvl1_vol
4530; GFX9-NEXT:  BB22_2:
4531; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
4532; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
4533; GFX9-NEXT:    v_readfirstlane_b32 s3, v1
4534; GFX9-NEXT:    v_mov_b32_e32 v1, 0
4535; GFX9-NEXT:    v_cndmask_b32_e64 v0, 5, 0, vcc
4536; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
4537; GFX9-NEXT:    v_mov_b32_e32 v1, s3
4538; GFX9-NEXT:    v_mov_b32_e32 v2, s2
4539; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
4540; GFX9-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
4541; GFX9-NEXT:    s_mov_b32 s3, 0xf000
4542; GFX9-NEXT:    s_mov_b32 s2, -1
4543; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
4544; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
4545; GFX9-NEXT:    s_endpgm
4546;
4547; GFX1064-LABEL: umax_i64_constant:
4548; GFX1064:       ; %bb.0: ; %entry
4549; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
4550; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
4551; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
4552; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
4553; GFX1064-NEXT:    ; implicit-def: $vgpr0_vgpr1
4554; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
4555; GFX1064-NEXT:    s_cbranch_execz BB22_2
4556; GFX1064-NEXT:  ; %bb.1:
4557; GFX1064-NEXT:    v_mov_b32_e32 v0, 5
4558; GFX1064-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
4559; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
4560; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4561; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
4562; GFX1064-NEXT:    ds_max_rtn_u64 v[0:1], v2, v[0:1]
4563; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4564; GFX1064-NEXT:    buffer_gl0_inv
4565; GFX1064-NEXT:    buffer_gl1_inv
4566; GFX1064-NEXT:  BB22_2:
4567; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
4568; GFX1064-NEXT:    s_or_b64 exec, exec, s[2:3]
4569; GFX1064-NEXT:    v_readfirstlane_b32 s2, v0
4570; GFX1064-NEXT:    v_readfirstlane_b32 s3, v1
4571; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
4572; GFX1064-NEXT:    v_cndmask_b32_e64 v0, 5, 0, vcc
4573; GFX1064-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
4574; GFX1064-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc
4575; GFX1064-NEXT:    v_cndmask_b32_e64 v1, 0, s3, vcc
4576; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
4577; GFX1064-NEXT:    s_mov_b32 s2, -1
4578; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
4579; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
4580; GFX1064-NEXT:    s_endpgm
4581;
4582; GFX1032-LABEL: umax_i64_constant:
4583; GFX1032:       ; %bb.0: ; %entry
4584; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
4585; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
4586; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
4587; GFX1032-NEXT:    ; implicit-def: $vgpr0_vgpr1
4588; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
4589; GFX1032-NEXT:    s_cbranch_execz BB22_2
4590; GFX1032-NEXT:  ; %bb.1:
4591; GFX1032-NEXT:    v_mov_b32_e32 v0, 5
4592; GFX1032-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
4593; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
4594; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4595; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
4596; GFX1032-NEXT:    ds_max_rtn_u64 v[0:1], v2, v[0:1]
4597; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4598; GFX1032-NEXT:    buffer_gl0_inv
4599; GFX1032-NEXT:    buffer_gl1_inv
4600; GFX1032-NEXT:  BB22_2:
4601; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
4602; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2
4603; GFX1032-NEXT:    v_readfirstlane_b32 s2, v0
4604; GFX1032-NEXT:    v_readfirstlane_b32 s3, v1
4605; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
4606; GFX1032-NEXT:    v_cndmask_b32_e64 v0, 5, 0, vcc_lo
4607; GFX1032-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[2:3], v[0:1]
4608; GFX1032-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
4609; GFX1032-NEXT:    v_cndmask_b32_e64 v1, 0, s3, vcc_lo
4610; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
4611; GFX1032-NEXT:    s_mov_b32 s2, -1
4612; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
4613; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
4614; GFX1032-NEXT:    s_endpgm
4615entry:
4616  %old = atomicrmw umax i64 addrspace(3)* @local_var64, i64 5 acq_rel
4617  store i64 %old, i64 addrspace(1)* %out
4618  ret void
4619}
4620
4621define amdgpu_kernel void @umin_i32_varying(i32 addrspace(1)* %out) {
4622;
4623;
4624; GFX7LESS-LABEL: umin_i32_varying:
4625; GFX7LESS:       ; %bb.0: ; %entry
4626; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
4627; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
4628; GFX7LESS-NEXT:    s_mov_b32 m0, -1
4629; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4630; GFX7LESS-NEXT:    ds_min_rtn_u32 v0, v1, v0
4631; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4632; GFX7LESS-NEXT:    buffer_wbinvl1
4633; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
4634; GFX7LESS-NEXT:    s_mov_b32 s2, -1
4635; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4636; GFX7LESS-NEXT:    s_endpgm
4637;
4638; GFX8-LABEL: umin_i32_varying:
4639; GFX8:       ; %bb.0: ; %entry
4640; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
4641; GFX8-NEXT:    v_mov_b32_e32 v2, v0
4642; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
4643; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
4644; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
4645; GFX8-NEXT:    v_mov_b32_e32 v1, -1
4646; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
4647; GFX8-NEXT:    s_not_b64 exec, exec
4648; GFX8-NEXT:    v_mov_b32_e32 v2, -1
4649; GFX8-NEXT:    s_not_b64 exec, exec
4650; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
4651; GFX8-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
4652; GFX8-NEXT:    s_nop 1
4653; GFX8-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
4654; GFX8-NEXT:    s_nop 1
4655; GFX8-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
4656; GFX8-NEXT:    s_nop 1
4657; GFX8-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
4658; GFX8-NEXT:    s_nop 1
4659; GFX8-NEXT:    v_min_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
4660; GFX8-NEXT:    s_nop 1
4661; GFX8-NEXT:    v_min_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
4662; GFX8-NEXT:    v_readlane_b32 s4, v2, 63
4663; GFX8-NEXT:    s_nop 0
4664; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
4665; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
4666; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
4667; GFX8-NEXT:    ; implicit-def: $vgpr0
4668; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
4669; GFX8-NEXT:    s_cbranch_execz BB23_2
4670; GFX8-NEXT:  ; %bb.1:
4671; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
4672; GFX8-NEXT:    v_mov_b32_e32 v3, s4
4673; GFX8-NEXT:    s_mov_b32 m0, -1
4674; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4675; GFX8-NEXT:    ds_min_rtn_u32 v0, v0, v3
4676; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4677; GFX8-NEXT:    buffer_wbinvl1_vol
4678; GFX8-NEXT:  BB23_2:
4679; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
4680; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
4681; GFX8-NEXT:    v_mov_b32_e32 v0, v1
4682; GFX8-NEXT:    v_min_u32_e32 v0, s2, v0
4683; GFX8-NEXT:    s_mov_b32 s3, 0xf000
4684; GFX8-NEXT:    s_mov_b32 s2, -1
4685; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
4686; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4687; GFX8-NEXT:    s_endpgm
4688;
4689; GFX9-LABEL: umin_i32_varying:
4690; GFX9:       ; %bb.0: ; %entry
4691; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
4692; GFX9-NEXT:    v_mov_b32_e32 v2, v0
4693; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
4694; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
4695; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
4696; GFX9-NEXT:    v_mov_b32_e32 v1, -1
4697; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
4698; GFX9-NEXT:    s_not_b64 exec, exec
4699; GFX9-NEXT:    v_mov_b32_e32 v2, -1
4700; GFX9-NEXT:    s_not_b64 exec, exec
4701; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
4702; GFX9-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
4703; GFX9-NEXT:    s_nop 1
4704; GFX9-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
4705; GFX9-NEXT:    s_nop 1
4706; GFX9-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
4707; GFX9-NEXT:    s_nop 1
4708; GFX9-NEXT:    v_min_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
4709; GFX9-NEXT:    s_nop 1
4710; GFX9-NEXT:    v_min_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
4711; GFX9-NEXT:    s_nop 1
4712; GFX9-NEXT:    v_min_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
4713; GFX9-NEXT:    v_readlane_b32 s4, v2, 63
4714; GFX9-NEXT:    s_nop 0
4715; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
4716; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
4717; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
4718; GFX9-NEXT:    ; implicit-def: $vgpr0
4719; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
4720; GFX9-NEXT:    s_cbranch_execz BB23_2
4721; GFX9-NEXT:  ; %bb.1:
4722; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
4723; GFX9-NEXT:    v_mov_b32_e32 v3, s4
4724; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4725; GFX9-NEXT:    ds_min_rtn_u32 v0, v0, v3
4726; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4727; GFX9-NEXT:    buffer_wbinvl1_vol
4728; GFX9-NEXT:  BB23_2:
4729; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
4730; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
4731; GFX9-NEXT:    v_mov_b32_e32 v0, v1
4732; GFX9-NEXT:    v_min_u32_e32 v0, s2, v0
4733; GFX9-NEXT:    s_mov_b32 s3, 0xf000
4734; GFX9-NEXT:    s_mov_b32 s2, -1
4735; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
4736; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4737; GFX9-NEXT:    s_endpgm
4738;
4739; GFX1064-LABEL: umin_i32_varying:
4740; GFX1064:       ; %bb.0: ; %entry
4741; GFX1064-NEXT:    v_mov_b32_e32 v1, v0
4742; GFX1064-NEXT:    s_not_b64 exec, exec
4743; GFX1064-NEXT:    v_mov_b32_e32 v1, -1
4744; GFX1064-NEXT:    s_not_b64 exec, exec
4745; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
4746; GFX1064-NEXT:    v_min_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
4747; GFX1064-NEXT:    v_mov_b32_e32 v3, -1
4748; GFX1064-NEXT:    v_min_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
4749; GFX1064-NEXT:    v_min_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
4750; GFX1064-NEXT:    v_min_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
4751; GFX1064-NEXT:    v_mov_b32_e32 v2, v1
4752; GFX1064-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
4753; GFX1064-NEXT:    v_min_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
4754; GFX1064-NEXT:    v_readlane_b32 s4, v1, 31
4755; GFX1064-NEXT:    v_mov_b32_e32 v2, s4
4756; GFX1064-NEXT:    v_min_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
4757; GFX1064-NEXT:    v_readlane_b32 s4, v1, 15
4758; GFX1064-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
4759; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
4760; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
4761; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
4762; GFX1064-NEXT:    v_readlane_b32 s5, v1, 31
4763; GFX1064-NEXT:    v_writelane_b32 v3, s4, 16
4764; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
4765; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
4766; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
4767; GFX1064-NEXT:    v_readlane_b32 s7, v1, 63
4768; GFX1064-NEXT:    v_readlane_b32 s6, v1, 47
4769; GFX1064-NEXT:    v_writelane_b32 v3, s5, 32
4770; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
4771; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
4772; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
4773; GFX1064-NEXT:    v_writelane_b32 v3, s6, 48
4774; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
4775; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
4776; GFX1064-NEXT:    s_mov_b32 s2, -1
4777; GFX1064-NEXT:    ; implicit-def: $vgpr0
4778; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
4779; GFX1064-NEXT:    s_cbranch_execz BB23_2
4780; GFX1064-NEXT:  ; %bb.1:
4781; GFX1064-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
4782; GFX1064-NEXT:    v_mov_b32_e32 v4, s7
4783; GFX1064-NEXT:    s_mov_b32 s3, s7
4784; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4785; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
4786; GFX1064-NEXT:    ds_min_rtn_u32 v0, v7, v4
4787; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4788; GFX1064-NEXT:    buffer_gl0_inv
4789; GFX1064-NEXT:    buffer_gl1_inv
4790; GFX1064-NEXT:  BB23_2:
4791; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
4792; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
4793; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
4794; GFX1064-NEXT:    v_mov_b32_e32 v0, v3
4795; GFX1064-NEXT:    v_min_u32_e32 v0, s3, v0
4796; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
4797; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
4798; GFX1064-NEXT:    s_nop 0
4799; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4800; GFX1064-NEXT:    s_endpgm
4801;
4802; GFX1032-LABEL: umin_i32_varying:
4803; GFX1032:       ; %bb.0: ; %entry
4804; GFX1032-NEXT:    v_mov_b32_e32 v1, v0
4805; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
4806; GFX1032-NEXT:    v_mov_b32_e32 v1, -1
4807; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
4808; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
4809; GFX1032-NEXT:    v_min_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
4810; GFX1032-NEXT:    v_min_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
4811; GFX1032-NEXT:    v_min_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
4812; GFX1032-NEXT:    v_min_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
4813; GFX1032-NEXT:    v_mov_b32_e32 v2, v1
4814; GFX1032-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
4815; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
4816; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
4817; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
4818; GFX1032-NEXT:    v_min_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
4819; GFX1032-NEXT:    v_mov_b32_e32 v3, -1
4820; GFX1032-NEXT:    v_readlane_b32 s3, v1, 15
4821; GFX1032-NEXT:    v_readlane_b32 s4, v1, 31
4822; GFX1032-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
4823; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
4824; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
4825; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
4826; GFX1032-NEXT:    v_writelane_b32 v3, s3, 16
4827; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
4828; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
4829; GFX1032-NEXT:    s_mov_b32 s2, -1
4830; GFX1032-NEXT:    ; implicit-def: $vgpr0
4831; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
4832; GFX1032-NEXT:    s_cbranch_execz BB23_2
4833; GFX1032-NEXT:  ; %bb.1:
4834; GFX1032-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
4835; GFX1032-NEXT:    v_mov_b32_e32 v4, s4
4836; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4837; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
4838; GFX1032-NEXT:    ds_min_rtn_u32 v0, v7, v4
4839; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4840; GFX1032-NEXT:    buffer_gl0_inv
4841; GFX1032-NEXT:    buffer_gl1_inv
4842; GFX1032-NEXT:  BB23_2:
4843; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
4844; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s3
4845; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
4846; GFX1032-NEXT:    v_mov_b32_e32 v0, v3
4847; GFX1032-NEXT:    v_min_u32_e32 v0, s3, v0
4848; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
4849; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
4850; GFX1032-NEXT:    s_nop 0
4851; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4852; GFX1032-NEXT:    s_endpgm
4853entry:
4854  %lane = call i32 @llvm.amdgcn.workitem.id.x()
4855  %old = atomicrmw umin i32 addrspace(3)* @local_var32, i32 %lane acq_rel
4856  store i32 %old, i32 addrspace(1)* %out
4857  ret void
4858}
4859
4860define amdgpu_kernel void @umin_i64_constant(i64 addrspace(1)* %out) {
4861;
4862;
4863; GFX7LESS-LABEL: umin_i64_constant:
4864; GFX7LESS:       ; %bb.0: ; %entry
4865; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
4866; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
4867; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
4868; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
4869; GFX7LESS-NEXT:    ; implicit-def: $vgpr0_vgpr1
4870; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
4871; GFX7LESS-NEXT:    s_cbranch_execz BB24_2
4872; GFX7LESS-NEXT:  ; %bb.1:
4873; GFX7LESS-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
4874; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 5
4875; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
4876; GFX7LESS-NEXT:    s_mov_b32 m0, -1
4877; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4878; GFX7LESS-NEXT:    ds_min_rtn_u64 v[0:1], v2, v[0:1]
4879; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4880; GFX7LESS-NEXT:    buffer_wbinvl1
4881; GFX7LESS-NEXT:  BB24_2:
4882; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
4883; GFX7LESS-NEXT:    v_readfirstlane_b32 s4, v0
4884; GFX7LESS-NEXT:    v_readfirstlane_b32 s5, v1
4885; GFX7LESS-NEXT:    s_mov_b32 s2, -1
4886; GFX7LESS-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc
4887; GFX7LESS-NEXT:    v_cndmask_b32_e64 v0, 5, -1, vcc
4888; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s5
4889; GFX7LESS-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
4890; GFX7LESS-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
4891; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s4
4892; GFX7LESS-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
4893; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
4894; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
4895; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
4896; GFX7LESS-NEXT:    s_endpgm
4897;
4898; GFX8-LABEL: umin_i64_constant:
4899; GFX8:       ; %bb.0: ; %entry
4900; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
4901; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
4902; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
4903; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
4904; GFX8-NEXT:    ; implicit-def: $vgpr0_vgpr1
4905; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
4906; GFX8-NEXT:    s_cbranch_execz BB24_2
4907; GFX8-NEXT:  ; %bb.1:
4908; GFX8-NEXT:    v_mov_b32_e32 v0, 5
4909; GFX8-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
4910; GFX8-NEXT:    v_mov_b32_e32 v1, 0
4911; GFX8-NEXT:    s_mov_b32 m0, -1
4912; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4913; GFX8-NEXT:    ds_min_rtn_u64 v[0:1], v2, v[0:1]
4914; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4915; GFX8-NEXT:    buffer_wbinvl1_vol
4916; GFX8-NEXT:  BB24_2:
4917; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
4918; GFX8-NEXT:    v_readfirstlane_b32 s5, v1
4919; GFX8-NEXT:    v_readfirstlane_b32 s4, v0
4920; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc
4921; GFX8-NEXT:    v_cndmask_b32_e64 v0, 5, -1, vcc
4922; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
4923; GFX8-NEXT:    v_mov_b32_e32 v2, s5
4924; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
4925; GFX8-NEXT:    v_mov_b32_e32 v2, s4
4926; GFX8-NEXT:    s_mov_b32 s2, -1
4927; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
4928; GFX8-NEXT:    s_mov_b32 s3, 0xf000
4929; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
4930; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
4931; GFX8-NEXT:    s_endpgm
4932;
4933; GFX9-LABEL: umin_i64_constant:
4934; GFX9:       ; %bb.0: ; %entry
4935; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
4936; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
4937; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
4938; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
4939; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1
4940; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
4941; GFX9-NEXT:    s_cbranch_execz BB24_2
4942; GFX9-NEXT:  ; %bb.1:
4943; GFX9-NEXT:    v_mov_b32_e32 v0, 5
4944; GFX9-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
4945; GFX9-NEXT:    v_mov_b32_e32 v1, 0
4946; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4947; GFX9-NEXT:    ds_min_rtn_u64 v[0:1], v2, v[0:1]
4948; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4949; GFX9-NEXT:    buffer_wbinvl1_vol
4950; GFX9-NEXT:  BB24_2:
4951; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
4952; GFX9-NEXT:    v_readfirstlane_b32 s5, v1
4953; GFX9-NEXT:    v_readfirstlane_b32 s4, v0
4954; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc
4955; GFX9-NEXT:    v_cndmask_b32_e64 v0, 5, -1, vcc
4956; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
4957; GFX9-NEXT:    v_mov_b32_e32 v2, s5
4958; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
4959; GFX9-NEXT:    v_mov_b32_e32 v2, s4
4960; GFX9-NEXT:    s_mov_b32 s2, -1
4961; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
4962; GFX9-NEXT:    s_mov_b32 s3, 0xf000
4963; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
4964; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
4965; GFX9-NEXT:    s_endpgm
4966;
4967; GFX1064-LABEL: umin_i64_constant:
4968; GFX1064:       ; %bb.0: ; %entry
4969; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
4970; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
4971; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
4972; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
4973; GFX1064-NEXT:    ; implicit-def: $vgpr0_vgpr1
4974; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
4975; GFX1064-NEXT:    s_cbranch_execz BB24_2
4976; GFX1064-NEXT:  ; %bb.1:
4977; GFX1064-NEXT:    v_mov_b32_e32 v0, 5
4978; GFX1064-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
4979; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
4980; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4981; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
4982; GFX1064-NEXT:    ds_min_rtn_u64 v[0:1], v2, v[0:1]
4983; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4984; GFX1064-NEXT:    buffer_gl0_inv
4985; GFX1064-NEXT:    buffer_gl1_inv
4986; GFX1064-NEXT:  BB24_2:
4987; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
4988; GFX1064-NEXT:    s_or_b64 exec, exec, s[2:3]
4989; GFX1064-NEXT:    v_readfirstlane_b32 s2, v0
4990; GFX1064-NEXT:    v_readfirstlane_b32 s3, v1
4991; GFX1064-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc
4992; GFX1064-NEXT:    v_cndmask_b32_e64 v0, 5, -1, vcc
4993; GFX1064-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
4994; GFX1064-NEXT:    v_cndmask_b32_e64 v1, v1, s3, vcc
4995; GFX1064-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc
4996; GFX1064-NEXT:    s_mov_b32 s2, -1
4997; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
4998; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
4999; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
5000; GFX1064-NEXT:    s_endpgm
5001;
5002; GFX1032-LABEL: umin_i64_constant:
5003; GFX1032:       ; %bb.0: ; %entry
5004; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
5005; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
5006; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
5007; GFX1032-NEXT:    ; implicit-def: $vgpr0_vgpr1
5008; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
5009; GFX1032-NEXT:    s_cbranch_execz BB24_2
5010; GFX1032-NEXT:  ; %bb.1:
5011; GFX1032-NEXT:    v_mov_b32_e32 v0, 5
5012; GFX1032-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
5013; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
5014; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5015; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
5016; GFX1032-NEXT:    ds_min_rtn_u64 v[0:1], v2, v[0:1]
5017; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5018; GFX1032-NEXT:    buffer_gl0_inv
5019; GFX1032-NEXT:    buffer_gl1_inv
5020; GFX1032-NEXT:  BB24_2:
5021; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
5022; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2
5023; GFX1032-NEXT:    v_readfirstlane_b32 s2, v0
5024; GFX1032-NEXT:    v_readfirstlane_b32 s3, v1
5025; GFX1032-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
5026; GFX1032-NEXT:    v_cndmask_b32_e64 v0, 5, -1, vcc_lo
5027; GFX1032-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[2:3], v[0:1]
5028; GFX1032-NEXT:    v_cndmask_b32_e64 v1, v1, s3, vcc_lo
5029; GFX1032-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
5030; GFX1032-NEXT:    s_mov_b32 s2, -1
5031; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
5032; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
5033; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
5034; GFX1032-NEXT:    s_endpgm
5035entry:
5036  %old = atomicrmw umin i64 addrspace(3)* @local_var64, i64 5 acq_rel
5037  store i64 %old, i64 addrspace(1)* %out
5038  ret void
5039}
5040