1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX900 %s
3; RUN: llc -march=amdgcn -mcpu=gfx908 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX908 %s
4; RUN: llc -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX90A %s
5; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX10 %s
6; RUN: llc -march=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX11 %s
7
8define amdgpu_kernel void @global_atomic_fadd_ret_f32(float addrspace(1)* %ptr) #0 {
9; GFX900-LABEL: global_atomic_fadd_ret_f32:
10; GFX900:       ; %bb.0:
11; GFX900-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
12; GFX900-NEXT:    s_mov_b64 s[2:3], 0
13; GFX900-NEXT:    v_mov_b32_e32 v0, 0
14; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
15; GFX900-NEXT:    s_load_dword s4, s[0:1], 0x0
16; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
17; GFX900-NEXT:    v_mov_b32_e32 v1, s4
18; GFX900-NEXT:  .LBB0_1: ; %atomicrmw.start
19; GFX900-NEXT:    ; =>This Inner Loop Header: Depth=1
20; GFX900-NEXT:    v_mov_b32_e32 v2, v1
21; GFX900-NEXT:    v_add_f32_e32 v1, 4.0, v2
22; GFX900-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
23; GFX900-NEXT:    global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
24; GFX900-NEXT:    s_waitcnt vmcnt(0)
25; GFX900-NEXT:    buffer_wbinvl1_vol
26; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
27; GFX900-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
28; GFX900-NEXT:    s_andn2_b64 exec, exec, s[2:3]
29; GFX900-NEXT:    s_cbranch_execnz .LBB0_1
30; GFX900-NEXT:  ; %bb.2: ; %atomicrmw.end
31; GFX900-NEXT:    s_or_b64 exec, exec, s[2:3]
32; GFX900-NEXT:    global_store_dword v[0:1], v1, off
33; GFX900-NEXT:    s_endpgm
34;
35; GFX908-LABEL: global_atomic_fadd_ret_f32:
36; GFX908:       ; %bb.0:
37; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
38; GFX908-NEXT:    s_mov_b64 s[2:3], 0
39; GFX908-NEXT:    v_mov_b32_e32 v0, 0
40; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
41; GFX908-NEXT:    s_load_dword s4, s[0:1], 0x0
42; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
43; GFX908-NEXT:    v_mov_b32_e32 v1, s4
44; GFX908-NEXT:  .LBB0_1: ; %atomicrmw.start
45; GFX908-NEXT:    ; =>This Inner Loop Header: Depth=1
46; GFX908-NEXT:    v_mov_b32_e32 v2, v1
47; GFX908-NEXT:    v_add_f32_e32 v1, 4.0, v2
48; GFX908-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
49; GFX908-NEXT:    global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
50; GFX908-NEXT:    s_waitcnt vmcnt(0)
51; GFX908-NEXT:    buffer_wbinvl1_vol
52; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
53; GFX908-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
54; GFX908-NEXT:    s_andn2_b64 exec, exec, s[2:3]
55; GFX908-NEXT:    s_cbranch_execnz .LBB0_1
56; GFX908-NEXT:  ; %bb.2: ; %atomicrmw.end
57; GFX908-NEXT:    s_or_b64 exec, exec, s[2:3]
58; GFX908-NEXT:    global_store_dword v[0:1], v1, off
59; GFX908-NEXT:    s_endpgm
60;
61; GFX90A-LABEL: global_atomic_fadd_ret_f32:
62; GFX90A:       ; %bb.0:
63; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
64; GFX90A-NEXT:    s_mov_b64 s[2:3], 0
65; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
66; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
67; GFX90A-NEXT:    s_load_dword s4, s[0:1], 0x0
68; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
69; GFX90A-NEXT:    v_mov_b32_e32 v1, s4
70; GFX90A-NEXT:  .LBB0_1: ; %atomicrmw.start
71; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
72; GFX90A-NEXT:    v_mov_b32_e32 v3, v1
73; GFX90A-NEXT:    v_add_f32_e32 v2, 4.0, v3
74; GFX90A-NEXT:    buffer_wbl2
75; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
76; GFX90A-NEXT:    global_atomic_cmpswap v1, v0, v[2:3], s[0:1] glc
77; GFX90A-NEXT:    s_waitcnt vmcnt(0)
78; GFX90A-NEXT:    buffer_invl2
79; GFX90A-NEXT:    buffer_wbinvl1_vol
80; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
81; GFX90A-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
82; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[2:3]
83; GFX90A-NEXT:    s_cbranch_execnz .LBB0_1
84; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
85; GFX90A-NEXT:    s_or_b64 exec, exec, s[2:3]
86; GFX90A-NEXT:    global_store_dword v[0:1], v1, off
87; GFX90A-NEXT:    s_endpgm
88;
89; GFX10-LABEL: global_atomic_fadd_ret_f32:
90; GFX10:       ; %bb.0:
91; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
92; GFX10-NEXT:    v_mov_b32_e32 v0, 0
93; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
94; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x0
95; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
96; GFX10-NEXT:    v_mov_b32_e32 v1, s2
97; GFX10-NEXT:    s_mov_b32 s2, 0
98; GFX10-NEXT:  .LBB0_1: ; %atomicrmw.start
99; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
100; GFX10-NEXT:    v_mov_b32_e32 v2, v1
101; GFX10-NEXT:    v_add_f32_e32 v1, 4.0, v2
102; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
103; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
104; GFX10-NEXT:    global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
105; GFX10-NEXT:    s_waitcnt vmcnt(0)
106; GFX10-NEXT:    buffer_gl0_inv
107; GFX10-NEXT:    buffer_gl1_inv
108; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
109; GFX10-NEXT:    s_or_b32 s2, vcc_lo, s2
110; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
111; GFX10-NEXT:    s_cbranch_execnz .LBB0_1
112; GFX10-NEXT:  ; %bb.2: ; %atomicrmw.end
113; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s2
114; GFX10-NEXT:    global_store_dword v[0:1], v1, off
115; GFX10-NEXT:    s_endpgm
116;
117; GFX11-LABEL: global_atomic_fadd_ret_f32:
118; GFX11:       ; %bb.0:
119; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
120; GFX11-NEXT:    v_mov_b32_e32 v0, 0
121; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
122; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x0
123; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
124; GFX11-NEXT:    v_mov_b32_e32 v1, s2
125; GFX11-NEXT:    s_mov_b32 s2, 0
126; GFX11-NEXT:  .LBB0_1: ; %atomicrmw.start
127; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
128; GFX11-NEXT:    v_mov_b32_e32 v2, v1
129; GFX11-NEXT:    v_add_f32_e32 v1, 4.0, v2
130; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
131; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
132; GFX11-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] glc
133; GFX11-NEXT:    s_waitcnt vmcnt(0)
134; GFX11-NEXT:    buffer_gl0_inv
135; GFX11-NEXT:    buffer_gl1_inv
136; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
137; GFX11-NEXT:    s_or_b32 s2, vcc_lo, s2
138; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
139; GFX11-NEXT:    s_cbranch_execnz .LBB0_1
140; GFX11-NEXT:  ; %bb.2: ; %atomicrmw.end
141; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s2
142; GFX11-NEXT:    global_store_b32 v[0:1], v1, off
143; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
144; GFX11-NEXT:    s_endpgm
145  %result = atomicrmw fadd float addrspace(1)* %ptr, float 4.0 seq_cst
146  store float %result, float addrspace(1)* undef
147  ret void
148}
149
150define amdgpu_kernel void @global_atomic_fadd_ret_f32_ieee(float addrspace(1)* %ptr) #2 {
151; GFX900-LABEL: global_atomic_fadd_ret_f32_ieee:
152; GFX900:       ; %bb.0:
153; GFX900-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
154; GFX900-NEXT:    s_mov_b64 s[2:3], 0
155; GFX900-NEXT:    v_mov_b32_e32 v0, 0
156; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
157; GFX900-NEXT:    s_load_dword s4, s[0:1], 0x0
158; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
159; GFX900-NEXT:    v_mov_b32_e32 v1, s4
160; GFX900-NEXT:  .LBB1_1: ; %atomicrmw.start
161; GFX900-NEXT:    ; =>This Inner Loop Header: Depth=1
162; GFX900-NEXT:    v_mov_b32_e32 v2, v1
163; GFX900-NEXT:    v_add_f32_e32 v1, 4.0, v2
164; GFX900-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
165; GFX900-NEXT:    global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
166; GFX900-NEXT:    s_waitcnt vmcnt(0)
167; GFX900-NEXT:    buffer_wbinvl1_vol
168; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
169; GFX900-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
170; GFX900-NEXT:    s_andn2_b64 exec, exec, s[2:3]
171; GFX900-NEXT:    s_cbranch_execnz .LBB1_1
172; GFX900-NEXT:  ; %bb.2: ; %atomicrmw.end
173; GFX900-NEXT:    s_or_b64 exec, exec, s[2:3]
174; GFX900-NEXT:    global_store_dword v[0:1], v1, off
175; GFX900-NEXT:    s_endpgm
176;
177; GFX908-LABEL: global_atomic_fadd_ret_f32_ieee:
178; GFX908:       ; %bb.0:
179; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
180; GFX908-NEXT:    s_mov_b64 s[2:3], 0
181; GFX908-NEXT:    v_mov_b32_e32 v0, 0
182; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
183; GFX908-NEXT:    s_load_dword s4, s[0:1], 0x0
184; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
185; GFX908-NEXT:    v_mov_b32_e32 v1, s4
186; GFX908-NEXT:  .LBB1_1: ; %atomicrmw.start
187; GFX908-NEXT:    ; =>This Inner Loop Header: Depth=1
188; GFX908-NEXT:    v_mov_b32_e32 v2, v1
189; GFX908-NEXT:    v_add_f32_e32 v1, 4.0, v2
190; GFX908-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
191; GFX908-NEXT:    global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
192; GFX908-NEXT:    s_waitcnt vmcnt(0)
193; GFX908-NEXT:    buffer_wbinvl1_vol
194; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
195; GFX908-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
196; GFX908-NEXT:    s_andn2_b64 exec, exec, s[2:3]
197; GFX908-NEXT:    s_cbranch_execnz .LBB1_1
198; GFX908-NEXT:  ; %bb.2: ; %atomicrmw.end
199; GFX908-NEXT:    s_or_b64 exec, exec, s[2:3]
200; GFX908-NEXT:    global_store_dword v[0:1], v1, off
201; GFX908-NEXT:    s_endpgm
202;
203; GFX90A-LABEL: global_atomic_fadd_ret_f32_ieee:
204; GFX90A:       ; %bb.0:
205; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
206; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
207; GFX90A-NEXT:    v_mov_b32_e32 v1, 4.0
208; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
209; GFX90A-NEXT:    global_atomic_add_f32 v0, v0, v1, s[0:1] glc
210; GFX90A-NEXT:    s_waitcnt vmcnt(0)
211; GFX90A-NEXT:    buffer_wbinvl1_vol
212; GFX90A-NEXT:    global_store_dword v[0:1], v0, off
213; GFX90A-NEXT:    s_endpgm
214;
215; GFX10-LABEL: global_atomic_fadd_ret_f32_ieee:
216; GFX10:       ; %bb.0:
217; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
218; GFX10-NEXT:    v_mov_b32_e32 v0, 0
219; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
220; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x0
221; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
222; GFX10-NEXT:    v_mov_b32_e32 v1, s2
223; GFX10-NEXT:    s_mov_b32 s2, 0
224; GFX10-NEXT:  .LBB1_1: ; %atomicrmw.start
225; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
226; GFX10-NEXT:    v_mov_b32_e32 v2, v1
227; GFX10-NEXT:    v_add_f32_e32 v1, 4.0, v2
228; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
229; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
230; GFX10-NEXT:    global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
231; GFX10-NEXT:    s_waitcnt vmcnt(0)
232; GFX10-NEXT:    buffer_gl0_inv
233; GFX10-NEXT:    buffer_gl1_inv
234; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
235; GFX10-NEXT:    s_or_b32 s2, vcc_lo, s2
236; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
237; GFX10-NEXT:    s_cbranch_execnz .LBB1_1
238; GFX10-NEXT:  ; %bb.2: ; %atomicrmw.end
239; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s2
240; GFX10-NEXT:    global_store_dword v[0:1], v1, off
241; GFX10-NEXT:    s_endpgm
242;
243; GFX11-LABEL: global_atomic_fadd_ret_f32_ieee:
244; GFX11:       ; %bb.0:
245; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
246; GFX11-NEXT:    v_mov_b32_e32 v0, 0
247; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
248; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x0
249; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
250; GFX11-NEXT:    v_mov_b32_e32 v1, s2
251; GFX11-NEXT:    s_mov_b32 s2, 0
252; GFX11-NEXT:  .LBB1_1: ; %atomicrmw.start
253; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
254; GFX11-NEXT:    v_mov_b32_e32 v2, v1
255; GFX11-NEXT:    v_add_f32_e32 v1, 4.0, v2
256; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
257; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
258; GFX11-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] glc
259; GFX11-NEXT:    s_waitcnt vmcnt(0)
260; GFX11-NEXT:    buffer_gl0_inv
261; GFX11-NEXT:    buffer_gl1_inv
262; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
263; GFX11-NEXT:    s_or_b32 s2, vcc_lo, s2
264; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
265; GFX11-NEXT:    s_cbranch_execnz .LBB1_1
266; GFX11-NEXT:  ; %bb.2: ; %atomicrmw.end
267; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s2
268; GFX11-NEXT:    global_store_b32 v[0:1], v1, off
269; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
270; GFX11-NEXT:    s_endpgm
271  %result = atomicrmw fadd float addrspace(1)* %ptr, float 4.0 syncscope("agent") seq_cst
272  store float %result, float addrspace(1)* undef
273  ret void
274}
275
276define amdgpu_kernel void @global_atomic_fadd_noret_f32(float addrspace(1)* %ptr) #0 {
277; GFX900-LABEL: global_atomic_fadd_noret_f32:
278; GFX900:       ; %bb.0:
279; GFX900-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
280; GFX900-NEXT:    s_mov_b64 s[2:3], 0
281; GFX900-NEXT:    v_mov_b32_e32 v2, 0
282; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
283; GFX900-NEXT:    s_load_dword s4, s[0:1], 0x0
284; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
285; GFX900-NEXT:    v_mov_b32_e32 v1, s4
286; GFX900-NEXT:  .LBB2_1: ; %atomicrmw.start
287; GFX900-NEXT:    ; =>This Inner Loop Header: Depth=1
288; GFX900-NEXT:    v_add_f32_e32 v0, 4.0, v1
289; GFX900-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
290; GFX900-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
291; GFX900-NEXT:    s_waitcnt vmcnt(0)
292; GFX900-NEXT:    buffer_wbinvl1_vol
293; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
294; GFX900-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
295; GFX900-NEXT:    v_mov_b32_e32 v1, v0
296; GFX900-NEXT:    s_andn2_b64 exec, exec, s[2:3]
297; GFX900-NEXT:    s_cbranch_execnz .LBB2_1
298; GFX900-NEXT:  ; %bb.2: ; %atomicrmw.end
299; GFX900-NEXT:    s_endpgm
300;
301; GFX908-LABEL: global_atomic_fadd_noret_f32:
302; GFX908:       ; %bb.0:
303; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
304; GFX908-NEXT:    v_mov_b32_e32 v0, 0
305; GFX908-NEXT:    v_mov_b32_e32 v1, 4.0
306; GFX908-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
307; GFX908-NEXT:    global_atomic_add_f32 v0, v1, s[0:1]
308; GFX908-NEXT:    s_waitcnt vmcnt(0)
309; GFX908-NEXT:    buffer_wbinvl1_vol
310; GFX908-NEXT:    s_endpgm
311;
312; GFX90A-LABEL: global_atomic_fadd_noret_f32:
313; GFX90A:       ; %bb.0:
314; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
315; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
316; GFX90A-NEXT:    v_mov_b32_e32 v1, 4.0
317; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
318; GFX90A-NEXT:    global_atomic_add_f32 v0, v1, s[0:1]
319; GFX90A-NEXT:    s_waitcnt vmcnt(0)
320; GFX90A-NEXT:    buffer_wbinvl1_vol
321; GFX90A-NEXT:    s_endpgm
322;
323; GFX10-LABEL: global_atomic_fadd_noret_f32:
324; GFX10:       ; %bb.0:
325; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
326; GFX10-NEXT:    v_mov_b32_e32 v2, 0
327; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
328; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x0
329; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
330; GFX10-NEXT:    v_mov_b32_e32 v1, s2
331; GFX10-NEXT:    s_mov_b32 s2, 0
332; GFX10-NEXT:  .LBB2_1: ; %atomicrmw.start
333; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
334; GFX10-NEXT:    v_add_f32_e32 v0, 4.0, v1
335; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
336; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
337; GFX10-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
338; GFX10-NEXT:    s_waitcnt vmcnt(0)
339; GFX10-NEXT:    buffer_gl0_inv
340; GFX10-NEXT:    buffer_gl1_inv
341; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v0, v1
342; GFX10-NEXT:    v_mov_b32_e32 v1, v0
343; GFX10-NEXT:    s_or_b32 s2, vcc_lo, s2
344; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
345; GFX10-NEXT:    s_cbranch_execnz .LBB2_1
346; GFX10-NEXT:  ; %bb.2: ; %atomicrmw.end
347; GFX10-NEXT:    s_endpgm
348;
349; GFX11-LABEL: global_atomic_fadd_noret_f32:
350; GFX11:       ; %bb.0:
351; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
352; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 4.0
353; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
354; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
355; GFX11-NEXT:    global_atomic_add_f32 v0, v1, s[0:1]
356; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
357; GFX11-NEXT:    buffer_gl0_inv
358; GFX11-NEXT:    buffer_gl1_inv
359; GFX11-NEXT:    s_endpgm
360  %result = atomicrmw fadd float addrspace(1)* %ptr, float 4.0 syncscope("agent") seq_cst
361  ret void
362}
363
364define amdgpu_kernel void @global_atomic_fadd_noret_f32_ieee(float addrspace(1)* %ptr) #2 {
365; GFX900-LABEL: global_atomic_fadd_noret_f32_ieee:
366; GFX900:       ; %bb.0:
367; GFX900-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
368; GFX900-NEXT:    s_mov_b64 s[2:3], 0
369; GFX900-NEXT:    v_mov_b32_e32 v2, 0
370; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
371; GFX900-NEXT:    s_load_dword s4, s[0:1], 0x0
372; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
373; GFX900-NEXT:    v_mov_b32_e32 v1, s4
374; GFX900-NEXT:  .LBB3_1: ; %atomicrmw.start
375; GFX900-NEXT:    ; =>This Inner Loop Header: Depth=1
376; GFX900-NEXT:    v_add_f32_e32 v0, 4.0, v1
377; GFX900-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
378; GFX900-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
379; GFX900-NEXT:    s_waitcnt vmcnt(0)
380; GFX900-NEXT:    buffer_wbinvl1_vol
381; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
382; GFX900-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
383; GFX900-NEXT:    v_mov_b32_e32 v1, v0
384; GFX900-NEXT:    s_andn2_b64 exec, exec, s[2:3]
385; GFX900-NEXT:    s_cbranch_execnz .LBB3_1
386; GFX900-NEXT:  ; %bb.2: ; %atomicrmw.end
387; GFX900-NEXT:    s_endpgm
388;
389; GFX908-LABEL: global_atomic_fadd_noret_f32_ieee:
390; GFX908:       ; %bb.0:
391; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
392; GFX908-NEXT:    v_mov_b32_e32 v0, 0
393; GFX908-NEXT:    v_mov_b32_e32 v1, 4.0
394; GFX908-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
395; GFX908-NEXT:    global_atomic_add_f32 v0, v1, s[0:1]
396; GFX908-NEXT:    s_waitcnt vmcnt(0)
397; GFX908-NEXT:    buffer_wbinvl1_vol
398; GFX908-NEXT:    s_endpgm
399;
400; GFX90A-LABEL: global_atomic_fadd_noret_f32_ieee:
401; GFX90A:       ; %bb.0:
402; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
403; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
404; GFX90A-NEXT:    v_mov_b32_e32 v1, 4.0
405; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
406; GFX90A-NEXT:    global_atomic_add_f32 v0, v1, s[0:1]
407; GFX90A-NEXT:    s_waitcnt vmcnt(0)
408; GFX90A-NEXT:    buffer_wbinvl1_vol
409; GFX90A-NEXT:    s_endpgm
410;
411; GFX10-LABEL: global_atomic_fadd_noret_f32_ieee:
412; GFX10:       ; %bb.0:
413; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
414; GFX10-NEXT:    v_mov_b32_e32 v2, 0
415; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
416; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x0
417; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
418; GFX10-NEXT:    v_mov_b32_e32 v1, s2
419; GFX10-NEXT:    s_mov_b32 s2, 0
420; GFX10-NEXT:  .LBB3_1: ; %atomicrmw.start
421; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
422; GFX10-NEXT:    v_add_f32_e32 v0, 4.0, v1
423; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
424; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
425; GFX10-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
426; GFX10-NEXT:    s_waitcnt vmcnt(0)
427; GFX10-NEXT:    buffer_gl0_inv
428; GFX10-NEXT:    buffer_gl1_inv
429; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v0, v1
430; GFX10-NEXT:    v_mov_b32_e32 v1, v0
431; GFX10-NEXT:    s_or_b32 s2, vcc_lo, s2
432; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
433; GFX10-NEXT:    s_cbranch_execnz .LBB3_1
434; GFX10-NEXT:  ; %bb.2: ; %atomicrmw.end
435; GFX10-NEXT:    s_endpgm
436;
437; GFX11-LABEL: global_atomic_fadd_noret_f32_ieee:
438; GFX11:       ; %bb.0:
439; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
440; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 4.0
441; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
442; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
443; GFX11-NEXT:    global_atomic_add_f32 v0, v1, s[0:1]
444; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
445; GFX11-NEXT:    buffer_gl0_inv
446; GFX11-NEXT:    buffer_gl1_inv
447; GFX11-NEXT:    s_endpgm
448  %result = atomicrmw fadd float addrspace(1)* %ptr, float 4.0 syncscope("agent") seq_cst
449  ret void
450}
451
452define amdgpu_kernel void @global_atomic_fadd_ret_f32_agent(float addrspace(1)* %ptr) #0 {
453; GFX900-LABEL: global_atomic_fadd_ret_f32_agent:
454; GFX900:       ; %bb.0:
455; GFX900-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
456; GFX900-NEXT:    s_mov_b64 s[2:3], 0
457; GFX900-NEXT:    v_mov_b32_e32 v0, 0
458; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
459; GFX900-NEXT:    s_load_dword s4, s[0:1], 0x0
460; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
461; GFX900-NEXT:    v_mov_b32_e32 v1, s4
462; GFX900-NEXT:  .LBB4_1: ; %atomicrmw.start
463; GFX900-NEXT:    ; =>This Inner Loop Header: Depth=1
464; GFX900-NEXT:    v_mov_b32_e32 v2, v1
465; GFX900-NEXT:    v_add_f32_e32 v1, 4.0, v2
466; GFX900-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
467; GFX900-NEXT:    global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
468; GFX900-NEXT:    s_waitcnt vmcnt(0)
469; GFX900-NEXT:    buffer_wbinvl1_vol
470; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
471; GFX900-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
472; GFX900-NEXT:    s_andn2_b64 exec, exec, s[2:3]
473; GFX900-NEXT:    s_cbranch_execnz .LBB4_1
474; GFX900-NEXT:  ; %bb.2: ; %atomicrmw.end
475; GFX900-NEXT:    s_or_b64 exec, exec, s[2:3]
476; GFX900-NEXT:    global_store_dword v[0:1], v1, off
477; GFX900-NEXT:    s_endpgm
478;
479; GFX908-LABEL: global_atomic_fadd_ret_f32_agent:
480; GFX908:       ; %bb.0:
481; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
482; GFX908-NEXT:    s_mov_b64 s[2:3], 0
483; GFX908-NEXT:    v_mov_b32_e32 v0, 0
484; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
485; GFX908-NEXT:    s_load_dword s4, s[0:1], 0x0
486; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
487; GFX908-NEXT:    v_mov_b32_e32 v1, s4
488; GFX908-NEXT:  .LBB4_1: ; %atomicrmw.start
489; GFX908-NEXT:    ; =>This Inner Loop Header: Depth=1
490; GFX908-NEXT:    v_mov_b32_e32 v2, v1
491; GFX908-NEXT:    v_add_f32_e32 v1, 4.0, v2
492; GFX908-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
493; GFX908-NEXT:    global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
494; GFX908-NEXT:    s_waitcnt vmcnt(0)
495; GFX908-NEXT:    buffer_wbinvl1_vol
496; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
497; GFX908-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
498; GFX908-NEXT:    s_andn2_b64 exec, exec, s[2:3]
499; GFX908-NEXT:    s_cbranch_execnz .LBB4_1
500; GFX908-NEXT:  ; %bb.2: ; %atomicrmw.end
501; GFX908-NEXT:    s_or_b64 exec, exec, s[2:3]
502; GFX908-NEXT:    global_store_dword v[0:1], v1, off
503; GFX908-NEXT:    s_endpgm
504;
505; GFX90A-LABEL: global_atomic_fadd_ret_f32_agent:
506; GFX90A:       ; %bb.0:
507; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
508; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
509; GFX90A-NEXT:    v_mov_b32_e32 v1, 4.0
510; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
511; GFX90A-NEXT:    global_atomic_add_f32 v0, v0, v1, s[0:1] glc
512; GFX90A-NEXT:    s_waitcnt vmcnt(0)
513; GFX90A-NEXT:    buffer_wbinvl1_vol
514; GFX90A-NEXT:    global_store_dword v[0:1], v0, off
515; GFX90A-NEXT:    s_endpgm
516;
517; GFX10-LABEL: global_atomic_fadd_ret_f32_agent:
518; GFX10:       ; %bb.0:
519; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
520; GFX10-NEXT:    v_mov_b32_e32 v0, 0
521; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
522; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x0
523; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
524; GFX10-NEXT:    v_mov_b32_e32 v1, s2
525; GFX10-NEXT:    s_mov_b32 s2, 0
526; GFX10-NEXT:  .LBB4_1: ; %atomicrmw.start
527; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
528; GFX10-NEXT:    v_mov_b32_e32 v2, v1
529; GFX10-NEXT:    v_add_f32_e32 v1, 4.0, v2
530; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
531; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
532; GFX10-NEXT:    global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
533; GFX10-NEXT:    s_waitcnt vmcnt(0)
534; GFX10-NEXT:    buffer_gl0_inv
535; GFX10-NEXT:    buffer_gl1_inv
536; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
537; GFX10-NEXT:    s_or_b32 s2, vcc_lo, s2
538; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
539; GFX10-NEXT:    s_cbranch_execnz .LBB4_1
540; GFX10-NEXT:  ; %bb.2: ; %atomicrmw.end
541; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s2
542; GFX10-NEXT:    global_store_dword v[0:1], v1, off
543; GFX10-NEXT:    s_endpgm
544;
545; GFX11-LABEL: global_atomic_fadd_ret_f32_agent:
546; GFX11:       ; %bb.0:
547; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
548; GFX11-NEXT:    v_mov_b32_e32 v0, 0
549; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
550; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x0
551; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
552; GFX11-NEXT:    v_mov_b32_e32 v1, s2
553; GFX11-NEXT:    s_mov_b32 s2, 0
554; GFX11-NEXT:  .LBB4_1: ; %atomicrmw.start
555; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
556; GFX11-NEXT:    v_mov_b32_e32 v2, v1
557; GFX11-NEXT:    v_add_f32_e32 v1, 4.0, v2
558; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
559; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
560; GFX11-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] glc
561; GFX11-NEXT:    s_waitcnt vmcnt(0)
562; GFX11-NEXT:    buffer_gl0_inv
563; GFX11-NEXT:    buffer_gl1_inv
564; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
565; GFX11-NEXT:    s_or_b32 s2, vcc_lo, s2
566; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
567; GFX11-NEXT:    s_cbranch_execnz .LBB4_1
568; GFX11-NEXT:  ; %bb.2: ; %atomicrmw.end
569; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s2
570; GFX11-NEXT:    global_store_b32 v[0:1], v1, off
571; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
572; GFX11-NEXT:    s_endpgm
573  %result = atomicrmw fadd float addrspace(1)* %ptr, float 4.0 syncscope("agent") seq_cst
574  store float %result, float addrspace(1)* undef
575  ret void
576}
577
578define amdgpu_kernel void @global_atomic_fadd_ret_f32_system(float addrspace(1)* %ptr) #0 {
579; GFX900-LABEL: global_atomic_fadd_ret_f32_system:
580; GFX900:       ; %bb.0:
581; GFX900-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
582; GFX900-NEXT:    s_mov_b64 s[2:3], 0
583; GFX900-NEXT:    v_mov_b32_e32 v0, 0
584; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
585; GFX900-NEXT:    s_load_dword s4, s[0:1], 0x0
586; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
587; GFX900-NEXT:    v_mov_b32_e32 v1, s4
588; GFX900-NEXT:  .LBB5_1: ; %atomicrmw.start
589; GFX900-NEXT:    ; =>This Inner Loop Header: Depth=1
590; GFX900-NEXT:    v_mov_b32_e32 v2, v1
591; GFX900-NEXT:    v_add_f32_e32 v1, 4.0, v2
592; GFX900-NEXT:    s_waitcnt vmcnt(0)
593; GFX900-NEXT:    global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
594; GFX900-NEXT:    s_waitcnt vmcnt(0)
595; GFX900-NEXT:    buffer_wbinvl1_vol
596; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
597; GFX900-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
598; GFX900-NEXT:    s_andn2_b64 exec, exec, s[2:3]
599; GFX900-NEXT:    s_cbranch_execnz .LBB5_1
600; GFX900-NEXT:  ; %bb.2: ; %atomicrmw.end
601; GFX900-NEXT:    s_or_b64 exec, exec, s[2:3]
602; GFX900-NEXT:    global_store_dword v[0:1], v1, off
603; GFX900-NEXT:    s_endpgm
604;
605; GFX908-LABEL: global_atomic_fadd_ret_f32_system:
606; GFX908:       ; %bb.0:
607; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
608; GFX908-NEXT:    s_mov_b64 s[2:3], 0
609; GFX908-NEXT:    v_mov_b32_e32 v0, 0
610; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
611; GFX908-NEXT:    s_load_dword s4, s[0:1], 0x0
612; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
613; GFX908-NEXT:    v_mov_b32_e32 v1, s4
614; GFX908-NEXT:  .LBB5_1: ; %atomicrmw.start
615; GFX908-NEXT:    ; =>This Inner Loop Header: Depth=1
616; GFX908-NEXT:    v_mov_b32_e32 v2, v1
617; GFX908-NEXT:    v_add_f32_e32 v1, 4.0, v2
618; GFX908-NEXT:    s_waitcnt vmcnt(0)
619; GFX908-NEXT:    global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
620; GFX908-NEXT:    s_waitcnt vmcnt(0)
621; GFX908-NEXT:    buffer_wbinvl1_vol
622; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
623; GFX908-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
624; GFX908-NEXT:    s_andn2_b64 exec, exec, s[2:3]
625; GFX908-NEXT:    s_cbranch_execnz .LBB5_1
626; GFX908-NEXT:  ; %bb.2: ; %atomicrmw.end
627; GFX908-NEXT:    s_or_b64 exec, exec, s[2:3]
628; GFX908-NEXT:    global_store_dword v[0:1], v1, off
629; GFX908-NEXT:    s_endpgm
630;
631; GFX90A-LABEL: global_atomic_fadd_ret_f32_system:
632; GFX90A:       ; %bb.0:
633; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
634; GFX90A-NEXT:    s_mov_b64 s[2:3], 0
635; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
636; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
637; GFX90A-NEXT:    s_load_dword s4, s[0:1], 0x0
638; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
639; GFX90A-NEXT:    v_mov_b32_e32 v1, s4
640; GFX90A-NEXT:  .LBB5_1: ; %atomicrmw.start
641; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
642; GFX90A-NEXT:    v_mov_b32_e32 v3, v1
643; GFX90A-NEXT:    v_add_f32_e32 v2, 4.0, v3
644; GFX90A-NEXT:    buffer_wbl2
645; GFX90A-NEXT:    s_waitcnt vmcnt(0)
646; GFX90A-NEXT:    global_atomic_cmpswap v1, v0, v[2:3], s[0:1] glc
647; GFX90A-NEXT:    s_waitcnt vmcnt(0)
648; GFX90A-NEXT:    buffer_invl2
649; GFX90A-NEXT:    buffer_wbinvl1_vol
650; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
651; GFX90A-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
652; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[2:3]
653; GFX90A-NEXT:    s_cbranch_execnz .LBB5_1
654; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
655; GFX90A-NEXT:    s_or_b64 exec, exec, s[2:3]
656; GFX90A-NEXT:    global_store_dword v[0:1], v1, off
657; GFX90A-NEXT:    s_endpgm
658;
659; GFX10-LABEL: global_atomic_fadd_ret_f32_system:
660; GFX10:       ; %bb.0:
661; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
662; GFX10-NEXT:    v_mov_b32_e32 v0, 0
663; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
664; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x0
665; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
666; GFX10-NEXT:    v_mov_b32_e32 v1, s2
667; GFX10-NEXT:    s_mov_b32 s2, 0
668; GFX10-NEXT:  .LBB5_1: ; %atomicrmw.start
669; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
670; GFX10-NEXT:    v_mov_b32_e32 v2, v1
671; GFX10-NEXT:    v_add_f32_e32 v1, 4.0, v2
672; GFX10-NEXT:    s_waitcnt vmcnt(0)
673; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
674; GFX10-NEXT:    global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
675; GFX10-NEXT:    s_waitcnt vmcnt(0)
676; GFX10-NEXT:    buffer_gl0_inv
677; GFX10-NEXT:    buffer_gl1_inv
678; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
679; GFX10-NEXT:    s_or_b32 s2, vcc_lo, s2
680; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
681; GFX10-NEXT:    s_cbranch_execnz .LBB5_1
682; GFX10-NEXT:  ; %bb.2: ; %atomicrmw.end
683; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s2
684; GFX10-NEXT:    global_store_dword v[0:1], v1, off
685; GFX10-NEXT:    s_endpgm
686;
687; GFX11-LABEL: global_atomic_fadd_ret_f32_system:
688; GFX11:       ; %bb.0:
689; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
690; GFX11-NEXT:    v_mov_b32_e32 v0, 0
691; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
692; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x0
693; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
694; GFX11-NEXT:    v_mov_b32_e32 v1, s2
695; GFX11-NEXT:    s_mov_b32 s2, 0
696; GFX11-NEXT:  .LBB5_1: ; %atomicrmw.start
697; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
698; GFX11-NEXT:    v_mov_b32_e32 v2, v1
699; GFX11-NEXT:    v_add_f32_e32 v1, 4.0, v2
700; GFX11-NEXT:    s_waitcnt vmcnt(0)
701; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
702; GFX11-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] glc
703; GFX11-NEXT:    s_waitcnt vmcnt(0)
704; GFX11-NEXT:    buffer_gl0_inv
705; GFX11-NEXT:    buffer_gl1_inv
706; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
707; GFX11-NEXT:    s_or_b32 s2, vcc_lo, s2
708; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
709; GFX11-NEXT:    s_cbranch_execnz .LBB5_1
710; GFX11-NEXT:  ; %bb.2: ; %atomicrmw.end
711; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s2
712; GFX11-NEXT:    global_store_b32 v[0:1], v1, off
713; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
714; GFX11-NEXT:    s_endpgm
715  %result = atomicrmw fadd float addrspace(1)* %ptr, float 4.0 syncscope("one-as") seq_cst
716  store float %result, float addrspace(1)* undef
717  ret void
718}
719
720define amdgpu_kernel void @global_atomic_fadd_ret_f32_wrong_subtarget(float addrspace(1)* %ptr) #1 {
721; GCN-LABEL: global_atomic_fadd_ret_f32_wrong_subtarget:
722; GCN:       ; %bb.0:
723; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
724; GCN-NEXT:    s_mov_b64 s[2:3], 0
725; GCN-NEXT:    v_mov_b32_e32 v0, 0
726; GCN-NEXT:    s_waitcnt lgkmcnt(0)
727; GCN-NEXT:    s_load_dword s4, s[0:1], 0x0
728; GCN-NEXT:    s_waitcnt lgkmcnt(0)
729; GCN-NEXT:    v_mov_b32_e32 v1, s4
730; GCN-NEXT:  .LBB6_1: ; %atomicrmw.start
731; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
732; GCN-NEXT:    v_mov_b32_e32 v2, v1
733; GCN-NEXT:    v_add_f32_e32 v1, 4.0, v2
734; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
735; GCN-NEXT:    global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
736; GCN-NEXT:    s_waitcnt vmcnt(0)
737; GCN-NEXT:    buffer_wbinvl1_vol
738; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
739; GCN-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
740; GCN-NEXT:    s_andn2_b64 exec, exec, s[2:3]
741; GCN-NEXT:    s_cbranch_execnz .LBB6_1
742; GCN-NEXT:  ; %bb.2: ; %atomicrmw.end
743; GCN-NEXT:    s_or_b64 exec, exec, s[2:3]
744; GCN-NEXT:    global_store_dword v[0:1], v1, off
745; GCN-NEXT:    s_endpgm
746  %result = atomicrmw fadd float addrspace(1)* %ptr, float 4.0 syncscope("agent") seq_cst
747  store float %result, float addrspace(1)* undef
748  ret void
749}
750
751define amdgpu_kernel void @global_atomic_fadd_noret_f32_wrong_subtarget(float addrspace(1)* %ptr) #1 {
752; GCN-LABEL: global_atomic_fadd_noret_f32_wrong_subtarget:
753; GCN:       ; %bb.0:
754; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
755; GCN-NEXT:    v_mov_b32_e32 v0, 0
756; GCN-NEXT:    v_mov_b32_e32 v1, 4.0
757; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
758; GCN-NEXT:    global_atomic_add_f32 v0, v1, s[0:1]
759; GCN-NEXT:    s_waitcnt vmcnt(0)
760; GCN-NEXT:    buffer_wbinvl1_vol
761; GCN-NEXT:    s_endpgm
762  %result = atomicrmw fadd float addrspace(1)* %ptr, float 4.0 syncscope("agent") seq_cst
763  ret void
764}
765
766define amdgpu_kernel void @global_atomic_fadd_noret_f32_safe(float addrspace(1)* %ptr) {
767; GFX900-LABEL: global_atomic_fadd_noret_f32_safe:
768; GFX900:       ; %bb.0:
769; GFX900-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
770; GFX900-NEXT:    s_mov_b64 s[2:3], 0
771; GFX900-NEXT:    v_mov_b32_e32 v2, 0
772; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
773; GFX900-NEXT:    s_load_dword s4, s[0:1], 0x0
774; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
775; GFX900-NEXT:    v_mov_b32_e32 v1, s4
776; GFX900-NEXT:  .LBB8_1: ; %atomicrmw.start
777; GFX900-NEXT:    ; =>This Inner Loop Header: Depth=1
778; GFX900-NEXT:    v_add_f32_e32 v0, 4.0, v1
779; GFX900-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
780; GFX900-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
781; GFX900-NEXT:    s_waitcnt vmcnt(0)
782; GFX900-NEXT:    buffer_wbinvl1_vol
783; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
784; GFX900-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
785; GFX900-NEXT:    v_mov_b32_e32 v1, v0
786; GFX900-NEXT:    s_andn2_b64 exec, exec, s[2:3]
787; GFX900-NEXT:    s_cbranch_execnz .LBB8_1
788; GFX900-NEXT:  ; %bb.2: ; %atomicrmw.end
789; GFX900-NEXT:    s_endpgm
790;
791; GFX908-LABEL: global_atomic_fadd_noret_f32_safe:
792; GFX908:       ; %bb.0:
793; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
794; GFX908-NEXT:    s_mov_b64 s[2:3], 0
795; GFX908-NEXT:    v_mov_b32_e32 v2, 0
796; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
797; GFX908-NEXT:    s_load_dword s4, s[0:1], 0x0
798; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
799; GFX908-NEXT:    v_mov_b32_e32 v1, s4
800; GFX908-NEXT:  .LBB8_1: ; %atomicrmw.start
801; GFX908-NEXT:    ; =>This Inner Loop Header: Depth=1
802; GFX908-NEXT:    v_add_f32_e32 v0, 4.0, v1
803; GFX908-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
804; GFX908-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
805; GFX908-NEXT:    s_waitcnt vmcnt(0)
806; GFX908-NEXT:    buffer_wbinvl1_vol
807; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
808; GFX908-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
809; GFX908-NEXT:    v_mov_b32_e32 v1, v0
810; GFX908-NEXT:    s_andn2_b64 exec, exec, s[2:3]
811; GFX908-NEXT:    s_cbranch_execnz .LBB8_1
812; GFX908-NEXT:  ; %bb.2: ; %atomicrmw.end
813; GFX908-NEXT:    s_endpgm
814;
815; GFX90A-LABEL: global_atomic_fadd_noret_f32_safe:
816; GFX90A:       ; %bb.0:
817; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
818; GFX90A-NEXT:    s_mov_b64 s[2:3], 0
819; GFX90A-NEXT:    v_mov_b32_e32 v2, 0
820; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
821; GFX90A-NEXT:    s_load_dword s4, s[0:1], 0x0
822; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
823; GFX90A-NEXT:    v_mov_b32_e32 v1, s4
824; GFX90A-NEXT:  .LBB8_1: ; %atomicrmw.start
825; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
826; GFX90A-NEXT:    v_add_f32_e32 v0, 4.0, v1
827; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
828; GFX90A-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
829; GFX90A-NEXT:    s_waitcnt vmcnt(0)
830; GFX90A-NEXT:    buffer_wbinvl1_vol
831; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
832; GFX90A-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
833; GFX90A-NEXT:    v_mov_b32_e32 v1, v0
834; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[2:3]
835; GFX90A-NEXT:    s_cbranch_execnz .LBB8_1
836; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
837; GFX90A-NEXT:    s_endpgm
838;
839; GFX10-LABEL: global_atomic_fadd_noret_f32_safe:
840; GFX10:       ; %bb.0:
841; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
842; GFX10-NEXT:    v_mov_b32_e32 v2, 0
843; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
844; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x0
845; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
846; GFX10-NEXT:    v_mov_b32_e32 v1, s2
847; GFX10-NEXT:    s_mov_b32 s2, 0
848; GFX10-NEXT:  .LBB8_1: ; %atomicrmw.start
849; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
850; GFX10-NEXT:    v_add_f32_e32 v0, 4.0, v1
851; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
852; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
853; GFX10-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
854; GFX10-NEXT:    s_waitcnt vmcnt(0)
855; GFX10-NEXT:    buffer_gl0_inv
856; GFX10-NEXT:    buffer_gl1_inv
857; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v0, v1
858; GFX10-NEXT:    v_mov_b32_e32 v1, v0
859; GFX10-NEXT:    s_or_b32 s2, vcc_lo, s2
860; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
861; GFX10-NEXT:    s_cbranch_execnz .LBB8_1
862; GFX10-NEXT:  ; %bb.2: ; %atomicrmw.end
863; GFX10-NEXT:    s_endpgm
864;
865; GFX11-LABEL: global_atomic_fadd_noret_f32_safe:
866; GFX11:       ; %bb.0:
867; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
868; GFX11-NEXT:    v_mov_b32_e32 v2, 0
869; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
870; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x0
871; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
872; GFX11-NEXT:    v_mov_b32_e32 v1, s2
873; GFX11-NEXT:    s_mov_b32 s2, 0
874; GFX11-NEXT:  .LBB8_1: ; %atomicrmw.start
875; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
876; GFX11-NEXT:    v_add_f32_e32 v0, 4.0, v1
877; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
878; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
879; GFX11-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
880; GFX11-NEXT:    s_waitcnt vmcnt(0)
881; GFX11-NEXT:    buffer_gl0_inv
882; GFX11-NEXT:    buffer_gl1_inv
883; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v0, v1
884; GFX11-NEXT:    v_mov_b32_e32 v1, v0
885; GFX11-NEXT:    s_or_b32 s2, vcc_lo, s2
886; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
887; GFX11-NEXT:    s_cbranch_execnz .LBB8_1
888; GFX11-NEXT:  ; %bb.2: ; %atomicrmw.end
889; GFX11-NEXT:    s_endpgm
890  %result = atomicrmw fadd float addrspace(1)* %ptr, float 4.0 syncscope("agent") seq_cst
891  ret void
892}
893
894define amdgpu_kernel void @infer_as_before_atomic(float* addrspace(4)* %arg) #0 {
895; GFX900-LABEL: infer_as_before_atomic:
896; GFX900:       ; %bb.0:
897; GFX900-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
898; GFX900-NEXT:    s_mov_b64 s[2:3], 0
899; GFX900-NEXT:    v_mov_b32_e32 v2, 0
900; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
901; GFX900-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
902; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
903; GFX900-NEXT:    s_load_dword s4, s[0:1], 0x0
904; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
905; GFX900-NEXT:    v_mov_b32_e32 v1, s4
906; GFX900-NEXT:  .LBB9_1: ; %atomicrmw.start
907; GFX900-NEXT:    ; =>This Inner Loop Header: Depth=1
908; GFX900-NEXT:    v_add_f32_e32 v0, 1.0, v1
909; GFX900-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
910; GFX900-NEXT:    s_waitcnt vmcnt(0)
911; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
912; GFX900-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
913; GFX900-NEXT:    v_mov_b32_e32 v1, v0
914; GFX900-NEXT:    s_andn2_b64 exec, exec, s[2:3]
915; GFX900-NEXT:    s_cbranch_execnz .LBB9_1
916; GFX900-NEXT:  ; %bb.2: ; %atomicrmw.end
917; GFX900-NEXT:    s_endpgm
918;
919; GFX908-LABEL: infer_as_before_atomic:
920; GFX908:       ; %bb.0:
921; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
922; GFX908-NEXT:    v_mov_b32_e32 v0, 0
923; GFX908-NEXT:    v_mov_b32_e32 v1, 1.0
924; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
925; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
926; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
927; GFX908-NEXT:    global_atomic_add_f32 v0, v1, s[0:1]
928; GFX908-NEXT:    s_endpgm
929;
930; GFX90A-LABEL: infer_as_before_atomic:
931; GFX90A:       ; %bb.0:
932; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
933; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
934; GFX90A-NEXT:    v_mov_b32_e32 v1, 1.0
935; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
936; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
937; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
938; GFX90A-NEXT:    global_atomic_add_f32 v0, v1, s[0:1]
939; GFX90A-NEXT:    s_endpgm
940;
941; GFX10-LABEL: infer_as_before_atomic:
942; GFX10:       ; %bb.0:
943; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
944; GFX10-NEXT:    v_mov_b32_e32 v2, 0
945; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
946; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
947; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
948; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x0
949; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
950; GFX10-NEXT:    v_mov_b32_e32 v1, s2
951; GFX10-NEXT:    s_mov_b32 s2, 0
952; GFX10-NEXT:  .LBB9_1: ; %atomicrmw.start
953; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
954; GFX10-NEXT:    v_add_f32_e32 v0, 1.0, v1
955; GFX10-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
956; GFX10-NEXT:    s_waitcnt vmcnt(0)
957; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v0, v1
958; GFX10-NEXT:    v_mov_b32_e32 v1, v0
959; GFX10-NEXT:    s_or_b32 s2, vcc_lo, s2
960; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
961; GFX10-NEXT:    s_cbranch_execnz .LBB9_1
962; GFX10-NEXT:  ; %bb.2: ; %atomicrmw.end
963; GFX10-NEXT:    s_endpgm
964;
965; GFX11-LABEL: infer_as_before_atomic:
966; GFX11:       ; %bb.0:
967; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
968; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 1.0
969; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
970; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
971; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
972; GFX11-NEXT:    global_atomic_add_f32 v0, v1, s[0:1]
973; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
974; GFX11-NEXT:    s_endpgm
975  %load = load float*, float* addrspace(4)* %arg
976  %v = atomicrmw fadd float* %load, float 1.0 syncscope("agent-one-as") monotonic, align 4
977  ret void
978}
979
980attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" }
981attributes #1 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "target-cpu"="gfx803" "target-features"="+atomic-fadd-no-rtn-insts" "amdgpu-unsafe-fp-atomics"="true" }
982attributes #2 = { "amdgpu-unsafe-fp-atomics"="true" }
983