1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
8
9define amdgpu_kernel void @flat_workgroup_unordered_load(
10; GFX7-LABEL: flat_workgroup_unordered_load:
11; GFX7:       ; %bb.0: ; %entry
12; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
13; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14; GFX7-NEXT:    v_mov_b32_e32 v0, s0
15; GFX7-NEXT:    v_mov_b32_e32 v1, s1
16; GFX7-NEXT:    flat_load_dword v0, v[0:1]
17; GFX7-NEXT:    v_mov_b32_e32 v2, s2
18; GFX7-NEXT:    v_mov_b32_e32 v3, s3
19; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
20; GFX7-NEXT:    flat_store_dword v[2:3], v0
21; GFX7-NEXT:    s_endpgm
22;
23; GFX10-WGP-LABEL: flat_workgroup_unordered_load:
24; GFX10-WGP:       ; %bb.0: ; %entry
25; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
26; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
27; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
28; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
29; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
30; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
31; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
32; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
33; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
34; GFX10-WGP-NEXT:    s_endpgm
35;
36; GFX10-CU-LABEL: flat_workgroup_unordered_load:
37; GFX10-CU:       ; %bb.0: ; %entry
38; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
39; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
40; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
41; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
42; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
43; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
44; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
45; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
46; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
47; GFX10-CU-NEXT:    s_endpgm
48;
49; SKIP-CACHE-INV-LABEL: flat_workgroup_unordered_load:
50; SKIP-CACHE-INV:       ; %bb.0: ; %entry
51; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
52; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
53; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
54; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
55; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
56; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
57; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
58; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
59; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
60; SKIP-CACHE-INV-NEXT:    s_endpgm
61;
62; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_unordered_load:
63; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
64; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
65; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
66; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
67; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
68; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1]
69; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
70; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
71; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
72; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
73; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
74;
75; GFX90A-TGSPLIT-LABEL: flat_workgroup_unordered_load:
76; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
77; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
78; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
79; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
80; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
81; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1]
82; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
83; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
84; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
85; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
86; GFX90A-TGSPLIT-NEXT:    s_endpgm
87    i32* %in, i32* %out) {
88entry:
89  %val = load atomic i32, i32* %in syncscope("workgroup") unordered, align 4
90  store i32 %val, i32* %out
91  ret void
92}
93
94define amdgpu_kernel void @flat_workgroup_monotonic_load(
95; GFX7-LABEL: flat_workgroup_monotonic_load:
96; GFX7:       ; %bb.0: ; %entry
97; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
98; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
99; GFX7-NEXT:    v_mov_b32_e32 v0, s0
100; GFX7-NEXT:    v_mov_b32_e32 v1, s1
101; GFX7-NEXT:    flat_load_dword v0, v[0:1]
102; GFX7-NEXT:    v_mov_b32_e32 v2, s2
103; GFX7-NEXT:    v_mov_b32_e32 v3, s3
104; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
105; GFX7-NEXT:    flat_store_dword v[2:3], v0
106; GFX7-NEXT:    s_endpgm
107;
108; GFX10-WGP-LABEL: flat_workgroup_monotonic_load:
109; GFX10-WGP:       ; %bb.0: ; %entry
110; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
111; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
112; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
113; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
114; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
115; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
116; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
117; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
118; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
119; GFX10-WGP-NEXT:    s_endpgm
120;
121; GFX10-CU-LABEL: flat_workgroup_monotonic_load:
122; GFX10-CU:       ; %bb.0: ; %entry
123; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
124; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
125; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
126; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
127; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
128; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
129; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
130; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
131; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
132; GFX10-CU-NEXT:    s_endpgm
133;
134; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_load:
135; SKIP-CACHE-INV:       ; %bb.0: ; %entry
136; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
137; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
138; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
139; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
140; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
141; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
142; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
143; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
144; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
145; SKIP-CACHE-INV-NEXT:    s_endpgm
146;
147; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_load:
148; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
149; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
150; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
151; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
152; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
153; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1]
154; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
155; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
156; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
157; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
158; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
159;
160; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_load:
161; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
162; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
163; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
164; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
165; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
166; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc
167; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
168; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
169; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
170; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
171; GFX90A-TGSPLIT-NEXT:    s_endpgm
172    i32* %in, i32* %out) {
173entry:
174  %val = load atomic i32, i32* %in syncscope("workgroup") monotonic, align 4
175  store i32 %val, i32* %out
176  ret void
177}
178
179define amdgpu_kernel void @flat_workgroup_acquire_load(
180; GFX7-LABEL: flat_workgroup_acquire_load:
181; GFX7:       ; %bb.0: ; %entry
182; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
183; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
184; GFX7-NEXT:    v_mov_b32_e32 v0, s0
185; GFX7-NEXT:    v_mov_b32_e32 v1, s1
186; GFX7-NEXT:    flat_load_dword v0, v[0:1]
187; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
188; GFX7-NEXT:    v_mov_b32_e32 v2, s2
189; GFX7-NEXT:    v_mov_b32_e32 v3, s3
190; GFX7-NEXT:    s_waitcnt vmcnt(0)
191; GFX7-NEXT:    flat_store_dword v[2:3], v0
192; GFX7-NEXT:    s_endpgm
193;
194; GFX10-WGP-LABEL: flat_workgroup_acquire_load:
195; GFX10-WGP:       ; %bb.0: ; %entry
196; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
197; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
198; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
199; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
200; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
201; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
202; GFX10-WGP-NEXT:    buffer_gl0_inv
203; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
204; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
205; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
206; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
207; GFX10-WGP-NEXT:    s_endpgm
208;
209; GFX10-CU-LABEL: flat_workgroup_acquire_load:
210; GFX10-CU:       ; %bb.0: ; %entry
211; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
212; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
213; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
214; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
215; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
216; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
217; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
218; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
219; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
220; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
221; GFX10-CU-NEXT:    s_endpgm
222;
223; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_load:
224; SKIP-CACHE-INV:       ; %bb.0: ; %entry
225; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
226; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
227; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
228; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
229; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
230; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
231; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
232; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
233; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
234; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
235; SKIP-CACHE-INV-NEXT:    s_endpgm
236;
237; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_load:
238; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
239; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
240; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
241; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
242; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
243; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1]
244; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
245; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
246; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
247; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
248; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
249; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
250;
251; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_load:
252; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
253; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
254; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
255; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
256; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
257; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc
258; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
259; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
260; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
261; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
262; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
263; GFX90A-TGSPLIT-NEXT:    s_endpgm
264    i32* %in, i32* %out) {
265entry:
266  %val = load atomic i32, i32* %in syncscope("workgroup") acquire, align 4
267  store i32 %val, i32* %out
268  ret void
269}
270
271define amdgpu_kernel void @flat_workgroup_seq_cst_load(
272; GFX7-LABEL: flat_workgroup_seq_cst_load:
273; GFX7:       ; %bb.0: ; %entry
274; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
275; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
276; GFX7-NEXT:    v_mov_b32_e32 v0, s0
277; GFX7-NEXT:    v_mov_b32_e32 v1, s1
278; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
279; GFX7-NEXT:    flat_load_dword v0, v[0:1]
280; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
281; GFX7-NEXT:    v_mov_b32_e32 v2, s2
282; GFX7-NEXT:    v_mov_b32_e32 v3, s3
283; GFX7-NEXT:    s_waitcnt vmcnt(0)
284; GFX7-NEXT:    flat_store_dword v[2:3], v0
285; GFX7-NEXT:    s_endpgm
286;
287; GFX10-WGP-LABEL: flat_workgroup_seq_cst_load:
288; GFX10-WGP:       ; %bb.0: ; %entry
289; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
290; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
291; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
292; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
293; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
294; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
295; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
296; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
297; GFX10-WGP-NEXT:    buffer_gl0_inv
298; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
299; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
300; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
301; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
302; GFX10-WGP-NEXT:    s_endpgm
303;
304; GFX10-CU-LABEL: flat_workgroup_seq_cst_load:
305; GFX10-CU:       ; %bb.0: ; %entry
306; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
307; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
308; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
309; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
310; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
311; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
312; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
313; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
314; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
315; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
316; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
317; GFX10-CU-NEXT:    s_endpgm
318;
319; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_load:
320; SKIP-CACHE-INV:       ; %bb.0: ; %entry
321; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
322; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
323; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
324; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
325; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
326; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
327; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
328; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
329; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
330; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
331; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
332; SKIP-CACHE-INV-NEXT:    s_endpgm
333;
334; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_load:
335; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
336; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
337; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
338; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
339; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
340; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
341; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1]
342; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
343; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
344; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
345; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
346; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
347; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
348;
349; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_load:
350; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
351; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
352; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
353; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
354; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
355; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
356; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc
357; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
358; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
359; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
360; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
361; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
362; GFX90A-TGSPLIT-NEXT:    s_endpgm
363    i32* %in, i32* %out) {
364entry:
365  %val = load atomic i32, i32* %in syncscope("workgroup") seq_cst, align 4
366  store i32 %val, i32* %out
367  ret void
368}
369
370define amdgpu_kernel void @flat_workgroup_unordered_store(
371; GFX7-LABEL: flat_workgroup_unordered_store:
372; GFX7:       ; %bb.0: ; %entry
373; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
374; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
375; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
376; GFX7-NEXT:    v_mov_b32_e32 v2, s2
377; GFX7-NEXT:    v_mov_b32_e32 v0, s0
378; GFX7-NEXT:    v_mov_b32_e32 v1, s1
379; GFX7-NEXT:    flat_store_dword v[0:1], v2
380; GFX7-NEXT:    s_endpgm
381;
382; GFX10-WGP-LABEL: flat_workgroup_unordered_store:
383; GFX10-WGP:       ; %bb.0: ; %entry
384; GFX10-WGP-NEXT:    s_clause 0x1
385; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
386; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
387; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
388; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
389; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
390; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
391; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
392; GFX10-WGP-NEXT:    s_endpgm
393;
394; GFX10-CU-LABEL: flat_workgroup_unordered_store:
395; GFX10-CU:       ; %bb.0: ; %entry
396; GFX10-CU-NEXT:    s_clause 0x1
397; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
398; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
399; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
400; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
401; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
402; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
403; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
404; GFX10-CU-NEXT:    s_endpgm
405;
406; SKIP-CACHE-INV-LABEL: flat_workgroup_unordered_store:
407; SKIP-CACHE-INV:       ; %bb.0: ; %entry
408; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
409; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
410; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
411; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
412; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
413; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
414; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
415; SKIP-CACHE-INV-NEXT:    s_endpgm
416;
417; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_unordered_store:
418; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
419; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
420; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
421; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
422; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
423; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
424; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
425; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
426;
427; GFX90A-TGSPLIT-LABEL: flat_workgroup_unordered_store:
428; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
429; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
430; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
431; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
432; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
433; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
434; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
435; GFX90A-TGSPLIT-NEXT:    s_endpgm
436    i32 %in, i32* %out) {
437entry:
438  store atomic i32 %in, i32* %out syncscope("workgroup") unordered, align 4
439  ret void
440}
441
442define amdgpu_kernel void @flat_workgroup_monotonic_store(
443; GFX7-LABEL: flat_workgroup_monotonic_store:
444; GFX7:       ; %bb.0: ; %entry
445; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
446; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
447; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
448; GFX7-NEXT:    v_mov_b32_e32 v2, s2
449; GFX7-NEXT:    v_mov_b32_e32 v0, s0
450; GFX7-NEXT:    v_mov_b32_e32 v1, s1
451; GFX7-NEXT:    flat_store_dword v[0:1], v2
452; GFX7-NEXT:    s_endpgm
453;
454; GFX10-WGP-LABEL: flat_workgroup_monotonic_store:
455; GFX10-WGP:       ; %bb.0: ; %entry
456; GFX10-WGP-NEXT:    s_clause 0x1
457; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
458; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
459; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
460; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
461; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
462; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
463; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
464; GFX10-WGP-NEXT:    s_endpgm
465;
466; GFX10-CU-LABEL: flat_workgroup_monotonic_store:
467; GFX10-CU:       ; %bb.0: ; %entry
468; GFX10-CU-NEXT:    s_clause 0x1
469; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
470; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
471; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
472; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
473; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
474; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
475; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
476; GFX10-CU-NEXT:    s_endpgm
477;
478; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_store:
479; SKIP-CACHE-INV:       ; %bb.0: ; %entry
480; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
481; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
482; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
483; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
484; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
485; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
486; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
487; SKIP-CACHE-INV-NEXT:    s_endpgm
488;
489; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_store:
490; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
491; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
492; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
493; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
494; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
495; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
496; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
497; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
498;
499; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_store:
500; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
501; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
502; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
503; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
504; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
505; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
506; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
507; GFX90A-TGSPLIT-NEXT:    s_endpgm
508    i32 %in, i32* %out) {
509entry:
510  store atomic i32 %in, i32* %out syncscope("workgroup") monotonic, align 4
511  ret void
512}
513
514define amdgpu_kernel void @flat_workgroup_release_store(
515; GFX7-LABEL: flat_workgroup_release_store:
516; GFX7:       ; %bb.0: ; %entry
517; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
518; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
519; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
520; GFX7-NEXT:    v_mov_b32_e32 v2, s2
521; GFX7-NEXT:    v_mov_b32_e32 v0, s0
522; GFX7-NEXT:    v_mov_b32_e32 v1, s1
523; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
524; GFX7-NEXT:    flat_store_dword v[0:1], v2
525; GFX7-NEXT:    s_endpgm
526;
527; GFX10-WGP-LABEL: flat_workgroup_release_store:
528; GFX10-WGP:       ; %bb.0: ; %entry
529; GFX10-WGP-NEXT:    s_clause 0x1
530; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
531; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
532; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
533; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
534; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
535; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
536; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
537; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
538; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
539; GFX10-WGP-NEXT:    s_endpgm
540;
541; GFX10-CU-LABEL: flat_workgroup_release_store:
542; GFX10-CU:       ; %bb.0: ; %entry
543; GFX10-CU-NEXT:    s_clause 0x1
544; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
545; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
546; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
547; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
548; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
549; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
550; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
551; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
552; GFX10-CU-NEXT:    s_endpgm
553;
554; SKIP-CACHE-INV-LABEL: flat_workgroup_release_store:
555; SKIP-CACHE-INV:       ; %bb.0: ; %entry
556; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
557; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
558; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
559; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
560; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
561; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
562; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
563; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
564; SKIP-CACHE-INV-NEXT:    s_endpgm
565;
566; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_store:
567; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
568; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
569; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
570; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
571; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
572; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
573; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
574; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
575; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
576;
577; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_store:
578; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
579; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
580; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
581; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
582; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
583; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
584; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
585; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
586; GFX90A-TGSPLIT-NEXT:    s_endpgm
587    i32 %in, i32* %out) {
588entry:
589  store atomic i32 %in, i32* %out syncscope("workgroup") release, align 4
590  ret void
591}
592
593define amdgpu_kernel void @flat_workgroup_seq_cst_store(
594; GFX7-LABEL: flat_workgroup_seq_cst_store:
595; GFX7:       ; %bb.0: ; %entry
596; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
597; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
598; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
599; GFX7-NEXT:    v_mov_b32_e32 v2, s2
600; GFX7-NEXT:    v_mov_b32_e32 v0, s0
601; GFX7-NEXT:    v_mov_b32_e32 v1, s1
602; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
603; GFX7-NEXT:    flat_store_dword v[0:1], v2
604; GFX7-NEXT:    s_endpgm
605;
606; GFX10-WGP-LABEL: flat_workgroup_seq_cst_store:
607; GFX10-WGP:       ; %bb.0: ; %entry
608; GFX10-WGP-NEXT:    s_clause 0x1
609; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
610; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
611; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
612; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
613; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
614; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
615; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
616; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
617; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
618; GFX10-WGP-NEXT:    s_endpgm
619;
620; GFX10-CU-LABEL: flat_workgroup_seq_cst_store:
621; GFX10-CU:       ; %bb.0: ; %entry
622; GFX10-CU-NEXT:    s_clause 0x1
623; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
624; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
625; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
626; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
627; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
628; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
629; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
630; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
631; GFX10-CU-NEXT:    s_endpgm
632;
633; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_store:
634; SKIP-CACHE-INV:       ; %bb.0: ; %entry
635; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
636; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
637; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
638; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
639; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
640; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
641; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
642; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
643; SKIP-CACHE-INV-NEXT:    s_endpgm
644;
645; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_store:
646; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
647; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
648; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
649; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
650; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
651; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
652; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
653; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
654; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
655;
656; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_store:
657; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
658; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
659; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
660; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
661; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
662; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
663; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
664; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
665; GFX90A-TGSPLIT-NEXT:    s_endpgm
666    i32 %in, i32* %out) {
667entry:
668  store atomic i32 %in, i32* %out syncscope("workgroup") seq_cst, align 4
669  ret void
670}
671
672define amdgpu_kernel void @flat_workgroup_monotonic_atomicrmw(
673; GFX7-LABEL: flat_workgroup_monotonic_atomicrmw:
674; GFX7:       ; %bb.0: ; %entry
675; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
676; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
677; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
678; GFX7-NEXT:    v_mov_b32_e32 v0, s0
679; GFX7-NEXT:    v_mov_b32_e32 v1, s1
680; GFX7-NEXT:    v_mov_b32_e32 v2, s2
681; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
682; GFX7-NEXT:    s_endpgm
683;
684; GFX10-WGP-LABEL: flat_workgroup_monotonic_atomicrmw:
685; GFX10-WGP:       ; %bb.0: ; %entry
686; GFX10-WGP-NEXT:    s_clause 0x1
687; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
688; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
689; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
690; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
691; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
692; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
693; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
694; GFX10-WGP-NEXT:    s_endpgm
695;
696; GFX10-CU-LABEL: flat_workgroup_monotonic_atomicrmw:
697; GFX10-CU:       ; %bb.0: ; %entry
698; GFX10-CU-NEXT:    s_clause 0x1
699; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
700; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
701; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
702; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
703; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
704; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
705; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
706; GFX10-CU-NEXT:    s_endpgm
707;
708; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_atomicrmw:
709; SKIP-CACHE-INV:       ; %bb.0: ; %entry
710; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
711; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
712; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
713; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
714; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
715; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
716; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
717; SKIP-CACHE-INV-NEXT:    s_endpgm
718;
719; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw:
720; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
721; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
722; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
723; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
724; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
725; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
726; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
727; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
728;
729; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw:
730; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
731; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
732; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
733; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
734; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
735; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
736; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
737; GFX90A-TGSPLIT-NEXT:    s_endpgm
738    i32* %out, i32 %in) {
739entry:
740  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") monotonic
741  ret void
742}
743
744define amdgpu_kernel void @flat_workgroup_acquire_atomicrmw(
745; GFX7-LABEL: flat_workgroup_acquire_atomicrmw:
746; GFX7:       ; %bb.0: ; %entry
747; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
748; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
749; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
750; GFX7-NEXT:    v_mov_b32_e32 v0, s0
751; GFX7-NEXT:    v_mov_b32_e32 v1, s1
752; GFX7-NEXT:    v_mov_b32_e32 v2, s2
753; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
754; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
755; GFX7-NEXT:    s_endpgm
756;
757; GFX10-WGP-LABEL: flat_workgroup_acquire_atomicrmw:
758; GFX10-WGP:       ; %bb.0: ; %entry
759; GFX10-WGP-NEXT:    s_clause 0x1
760; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
761; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
762; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
763; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
764; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
765; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
766; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
767; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
768; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
769; GFX10-WGP-NEXT:    buffer_gl0_inv
770; GFX10-WGP-NEXT:    s_endpgm
771;
772; GFX10-CU-LABEL: flat_workgroup_acquire_atomicrmw:
773; GFX10-CU:       ; %bb.0: ; %entry
774; GFX10-CU-NEXT:    s_clause 0x1
775; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
776; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
777; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
778; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
779; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
780; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
781; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
782; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
783; GFX10-CU-NEXT:    s_endpgm
784;
785; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_atomicrmw:
786; SKIP-CACHE-INV:       ; %bb.0: ; %entry
787; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
788; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
789; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
790; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
791; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
792; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
793; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
794; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
795; SKIP-CACHE-INV-NEXT:    s_endpgm
796;
797; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw:
798; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
799; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
800; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
801; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
802; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
803; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
804; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
805; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
806; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
807;
808; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw:
809; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
810; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
811; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
812; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
813; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
814; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
815; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
816; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
817; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
818; GFX90A-TGSPLIT-NEXT:    s_endpgm
819    i32* %out, i32 %in) {
820entry:
821  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acquire
822  ret void
823}
824
825define amdgpu_kernel void @flat_workgroup_release_atomicrmw(
826; GFX7-LABEL: flat_workgroup_release_atomicrmw:
827; GFX7:       ; %bb.0: ; %entry
828; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
829; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
830; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
831; GFX7-NEXT:    v_mov_b32_e32 v0, s0
832; GFX7-NEXT:    v_mov_b32_e32 v1, s1
833; GFX7-NEXT:    v_mov_b32_e32 v2, s2
834; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
835; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
836; GFX7-NEXT:    s_endpgm
837;
838; GFX10-WGP-LABEL: flat_workgroup_release_atomicrmw:
839; GFX10-WGP:       ; %bb.0: ; %entry
840; GFX10-WGP-NEXT:    s_clause 0x1
841; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
842; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
843; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
844; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
845; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
846; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
847; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
848; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
849; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
850; GFX10-WGP-NEXT:    s_endpgm
851;
852; GFX10-CU-LABEL: flat_workgroup_release_atomicrmw:
853; GFX10-CU:       ; %bb.0: ; %entry
854; GFX10-CU-NEXT:    s_clause 0x1
855; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
856; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
857; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
858; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
859; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
860; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
861; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
862; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
863; GFX10-CU-NEXT:    s_endpgm
864;
865; SKIP-CACHE-INV-LABEL: flat_workgroup_release_atomicrmw:
866; SKIP-CACHE-INV:       ; %bb.0: ; %entry
867; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
868; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
869; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
870; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
871; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
872; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
873; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
874; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
875; SKIP-CACHE-INV-NEXT:    s_endpgm
876;
877; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_atomicrmw:
878; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
879; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
880; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
881; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
882; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
883; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
884; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
885; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
886; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
887;
888; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_atomicrmw:
889; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
890; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
891; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
892; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
893; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
894; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
895; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
896; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
897; GFX90A-TGSPLIT-NEXT:    s_endpgm
898    i32* %out, i32 %in) {
899entry:
900  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") release
901  ret void
902}
903
904define amdgpu_kernel void @flat_workgroup_acq_rel_atomicrmw(
905; GFX7-LABEL: flat_workgroup_acq_rel_atomicrmw:
906; GFX7:       ; %bb.0: ; %entry
907; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
908; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
909; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
910; GFX7-NEXT:    v_mov_b32_e32 v0, s0
911; GFX7-NEXT:    v_mov_b32_e32 v1, s1
912; GFX7-NEXT:    v_mov_b32_e32 v2, s2
913; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
914; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
915; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
916; GFX7-NEXT:    s_endpgm
917;
918; GFX10-WGP-LABEL: flat_workgroup_acq_rel_atomicrmw:
919; GFX10-WGP:       ; %bb.0: ; %entry
920; GFX10-WGP-NEXT:    s_clause 0x1
921; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
922; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
923; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
924; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
925; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
926; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
927; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
928; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
929; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
930; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
931; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
932; GFX10-WGP-NEXT:    buffer_gl0_inv
933; GFX10-WGP-NEXT:    s_endpgm
934;
935; GFX10-CU-LABEL: flat_workgroup_acq_rel_atomicrmw:
936; GFX10-CU:       ; %bb.0: ; %entry
937; GFX10-CU-NEXT:    s_clause 0x1
938; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
939; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
940; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
941; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
942; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
943; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
944; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
945; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
946; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
947; GFX10-CU-NEXT:    s_endpgm
948;
949; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_atomicrmw:
950; SKIP-CACHE-INV:       ; %bb.0: ; %entry
951; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
952; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
953; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
954; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
955; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
956; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
957; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
958; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
959; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
960; SKIP-CACHE-INV-NEXT:    s_endpgm
961;
962; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw:
963; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
964; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
965; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
966; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
967; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
968; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
969; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
970; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
971; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
972; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
973;
974; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw:
975; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
976; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
977; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
978; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
979; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
980; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
981; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
982; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
983; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
984; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
985; GFX90A-TGSPLIT-NEXT:    s_endpgm
986    i32* %out, i32 %in) {
987entry:
988  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acq_rel
989  ret void
990}
991
992define amdgpu_kernel void @flat_workgroup_seq_cst_atomicrmw(
993; GFX7-LABEL: flat_workgroup_seq_cst_atomicrmw:
994; GFX7:       ; %bb.0: ; %entry
995; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
996; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
997; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
998; GFX7-NEXT:    v_mov_b32_e32 v0, s0
999; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1000; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1001; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1002; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1003; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1004; GFX7-NEXT:    s_endpgm
1005;
1006; GFX10-WGP-LABEL: flat_workgroup_seq_cst_atomicrmw:
1007; GFX10-WGP:       ; %bb.0: ; %entry
1008; GFX10-WGP-NEXT:    s_clause 0x1
1009; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1010; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1011; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1012; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1013; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1014; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1015; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1016; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1017; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1018; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1019; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1020; GFX10-WGP-NEXT:    buffer_gl0_inv
1021; GFX10-WGP-NEXT:    s_endpgm
1022;
1023; GFX10-CU-LABEL: flat_workgroup_seq_cst_atomicrmw:
1024; GFX10-CU:       ; %bb.0: ; %entry
1025; GFX10-CU-NEXT:    s_clause 0x1
1026; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1027; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1028; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1029; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1030; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1031; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1032; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1033; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1034; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1035; GFX10-CU-NEXT:    s_endpgm
1036;
1037; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_atomicrmw:
1038; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1039; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1040; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1041; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1042; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1043; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1044; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1045; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1046; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1047; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1048; SKIP-CACHE-INV-NEXT:    s_endpgm
1049;
1050; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw:
1051; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1052; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1053; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1054; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1055; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1056; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1057; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1058; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1059; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1060; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1061;
1062; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw:
1063; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1064; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1065; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1066; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1067; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1068; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1069; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1070; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1071; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1072; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1073; GFX90A-TGSPLIT-NEXT:    s_endpgm
1074    i32* %out, i32 %in) {
1075entry:
1076  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") seq_cst
1077  ret void
1078}
1079
1080define amdgpu_kernel void @flat_workgroup_acquire_ret_atomicrmw(
1081; GFX7-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1082; GFX7:       ; %bb.0: ; %entry
1083; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1084; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1085; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1086; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1087; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1088; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1089; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1090; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1091; GFX7-NEXT:    flat_store_dword v[0:1], v2
1092; GFX7-NEXT:    s_endpgm
1093;
1094; GFX10-WGP-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1095; GFX10-WGP:       ; %bb.0: ; %entry
1096; GFX10-WGP-NEXT:    s_clause 0x1
1097; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1098; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1099; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1100; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1101; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1102; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1103; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1104; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1105; GFX10-WGP-NEXT:    buffer_gl0_inv
1106; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1107; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1108; GFX10-WGP-NEXT:    s_endpgm
1109;
1110; GFX10-CU-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1111; GFX10-CU:       ; %bb.0: ; %entry
1112; GFX10-CU-NEXT:    s_clause 0x1
1113; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1114; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1115; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1116; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1117; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1118; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1119; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1120; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1121; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1122; GFX10-CU-NEXT:    s_endpgm
1123;
1124; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1125; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1126; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1127; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1128; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1129; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1130; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1131; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1132; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1133; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1134; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1135; SKIP-CACHE-INV-NEXT:    s_endpgm
1136;
1137; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1138; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1139; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1140; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1141; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1142; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1143; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1144; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1145; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1146; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1147; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1148;
1149; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1150; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1151; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1152; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1153; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1154; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1155; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1156; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1157; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1158; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1159; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1160; GFX90A-TGSPLIT-NEXT:    s_endpgm
1161    i32* %out, i32 %in) {
1162entry:
1163  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acquire
1164  store i32 %val, i32* %out, align 4
1165  ret void
1166}
1167
1168define amdgpu_kernel void @flat_workgroup_acq_rel_ret_atomicrmw(
1169; GFX7-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1170; GFX7:       ; %bb.0: ; %entry
1171; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1172; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1173; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1174; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1175; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1176; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1177; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1178; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1179; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1180; GFX7-NEXT:    flat_store_dword v[0:1], v2
1181; GFX7-NEXT:    s_endpgm
1182;
1183; GFX10-WGP-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1184; GFX10-WGP:       ; %bb.0: ; %entry
1185; GFX10-WGP-NEXT:    s_clause 0x1
1186; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1187; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1188; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1189; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1190; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1191; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1192; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1193; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1194; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1195; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1196; GFX10-WGP-NEXT:    buffer_gl0_inv
1197; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1198; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1199; GFX10-WGP-NEXT:    s_endpgm
1200;
1201; GFX10-CU-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1202; GFX10-CU:       ; %bb.0: ; %entry
1203; GFX10-CU-NEXT:    s_clause 0x1
1204; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1205; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1206; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1207; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1208; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1209; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1210; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1211; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1212; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1213; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1214; GFX10-CU-NEXT:    s_endpgm
1215;
1216; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1217; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1218; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1219; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1220; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1221; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1222; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1223; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1224; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1225; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1226; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1227; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1228; SKIP-CACHE-INV-NEXT:    s_endpgm
1229;
1230; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1231; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1232; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1233; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1234; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1235; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1236; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1237; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1238; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1239; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1240; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1241; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1242;
1243; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1244; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1245; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1246; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1247; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1248; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1249; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1250; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1251; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1252; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1253; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1254; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1255; GFX90A-TGSPLIT-NEXT:    s_endpgm
1256    i32* %out, i32 %in) {
1257entry:
1258  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acq_rel
1259  store i32 %val, i32* %out, align 4
1260  ret void
1261}
1262
1263define amdgpu_kernel void @flat_workgroup_seq_cst_ret_atomicrmw(
1264; GFX7-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
1265; GFX7:       ; %bb.0: ; %entry
1266; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1267; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1268; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1269; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1270; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1271; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1272; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1273; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1274; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1275; GFX7-NEXT:    flat_store_dword v[0:1], v2
1276; GFX7-NEXT:    s_endpgm
1277;
1278; GFX10-WGP-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
1279; GFX10-WGP:       ; %bb.0: ; %entry
1280; GFX10-WGP-NEXT:    s_clause 0x1
1281; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1282; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1283; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1284; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1285; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1286; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1287; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1288; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1289; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1290; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1291; GFX10-WGP-NEXT:    buffer_gl0_inv
1292; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1293; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1294; GFX10-WGP-NEXT:    s_endpgm
1295;
1296; GFX10-CU-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
1297; GFX10-CU:       ; %bb.0: ; %entry
1298; GFX10-CU-NEXT:    s_clause 0x1
1299; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1300; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1301; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1302; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1303; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1304; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1305; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1306; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1307; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1308; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1309; GFX10-CU-NEXT:    s_endpgm
1310;
1311; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
1312; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1313; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1314; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1315; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1316; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1317; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1318; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1319; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1320; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1321; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1322; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1323; SKIP-CACHE-INV-NEXT:    s_endpgm
1324;
1325; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
1326; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1327; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1328; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1329; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1330; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1331; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1332; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1333; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1334; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1335; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1336; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1337;
1338; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
1339; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1340; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1341; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1342; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1343; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1344; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1345; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1346; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1347; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1348; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1349; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1350; GFX90A-TGSPLIT-NEXT:    s_endpgm
1351    i32* %out, i32 %in) {
1352entry:
1353  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") seq_cst
1354  store i32 %val, i32* %out, align 4
1355  ret void
1356}
1357
1358define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_cmpxchg(
1359; GFX7-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
1360; GFX7:       ; %bb.0: ; %entry
1361; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1362; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1363; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1364; GFX7-NEXT:    s_add_u32 s0, s0, 16
1365; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1366; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1367; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1368; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1369; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1370; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1371; GFX7-NEXT:    s_endpgm
1372;
1373; GFX10-WGP-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
1374; GFX10-WGP:       ; %bb.0: ; %entry
1375; GFX10-WGP-NEXT:    s_clause 0x1
1376; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1377; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1378; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1379; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1380; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1381; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1382; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1383; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1384; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1385; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1386; GFX10-WGP-NEXT:    s_endpgm
1387;
1388; GFX10-CU-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
1389; GFX10-CU:       ; %bb.0: ; %entry
1390; GFX10-CU-NEXT:    s_clause 0x1
1391; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1392; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1393; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1394; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1395; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1396; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1397; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1398; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1399; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1400; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1401; GFX10-CU-NEXT:    s_endpgm
1402;
1403; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
1404; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1405; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1406; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1407; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1408; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1409; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1410; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1411; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1412; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1413; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1414; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1415; SKIP-CACHE-INV-NEXT:    s_endpgm
1416;
1417; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
1418; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1419; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1420; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1421; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1422; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1423; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1424; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1425; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1426;
1427; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
1428; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1429; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1430; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1431; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1432; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1433; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1434; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1435; GFX90A-TGSPLIT-NEXT:    s_endpgm
1436    i32* %out, i32 %in, i32 %old) {
1437entry:
1438  %gep = getelementptr i32, i32* %out, i32 4
1439  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic
1440  ret void
1441}
1442
1443define amdgpu_kernel void @flat_workgroup_acquire_monotonic_cmpxchg(
1444; GFX7-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
1445; GFX7:       ; %bb.0: ; %entry
1446; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1447; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1448; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1449; GFX7-NEXT:    s_add_u32 s0, s0, 16
1450; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1451; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1452; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1453; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1454; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1455; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1456; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1457; GFX7-NEXT:    s_endpgm
1458;
1459; GFX10-WGP-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
1460; GFX10-WGP:       ; %bb.0: ; %entry
1461; GFX10-WGP-NEXT:    s_clause 0x1
1462; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1463; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1464; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1465; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1466; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1467; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1468; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1469; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1470; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1471; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1472; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1473; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1474; GFX10-WGP-NEXT:    buffer_gl0_inv
1475; GFX10-WGP-NEXT:    s_endpgm
1476;
1477; GFX10-CU-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
1478; GFX10-CU:       ; %bb.0: ; %entry
1479; GFX10-CU-NEXT:    s_clause 0x1
1480; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1481; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1482; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1483; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1484; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1485; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1486; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1487; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1488; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1489; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1490; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1491; GFX10-CU-NEXT:    s_endpgm
1492;
1493; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
1494; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1495; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1496; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1497; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1498; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1499; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1500; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1501; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1502; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1503; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1504; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1505; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1506; SKIP-CACHE-INV-NEXT:    s_endpgm
1507;
1508; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
1509; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1510; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1511; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1512; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1513; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1514; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1515; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1516; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1517; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1518;
1519; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
1520; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1521; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1522; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1523; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1524; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1525; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1526; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1527; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1528; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1529; GFX90A-TGSPLIT-NEXT:    s_endpgm
1530    i32* %out, i32 %in, i32 %old) {
1531entry:
1532  %gep = getelementptr i32, i32* %out, i32 4
1533  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
1534  ret void
1535}
1536
1537define amdgpu_kernel void @flat_workgroup_release_monotonic_cmpxchg(
1538; GFX7-LABEL: flat_workgroup_release_monotonic_cmpxchg:
1539; GFX7:       ; %bb.0: ; %entry
1540; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1541; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1542; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1543; GFX7-NEXT:    s_add_u32 s0, s0, 16
1544; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1545; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1546; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1547; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1548; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1549; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1550; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1551; GFX7-NEXT:    s_endpgm
1552;
1553; GFX10-WGP-LABEL: flat_workgroup_release_monotonic_cmpxchg:
1554; GFX10-WGP:       ; %bb.0: ; %entry
1555; GFX10-WGP-NEXT:    s_clause 0x1
1556; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1557; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1558; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1559; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1560; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1561; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1562; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1563; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1564; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1565; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1566; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1567; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1568; GFX10-WGP-NEXT:    s_endpgm
1569;
1570; GFX10-CU-LABEL: flat_workgroup_release_monotonic_cmpxchg:
1571; GFX10-CU:       ; %bb.0: ; %entry
1572; GFX10-CU-NEXT:    s_clause 0x1
1573; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1574; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1575; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1576; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1577; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1578; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1579; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1580; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1581; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1582; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1583; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1584; GFX10-CU-NEXT:    s_endpgm
1585;
1586; SKIP-CACHE-INV-LABEL: flat_workgroup_release_monotonic_cmpxchg:
1587; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1588; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1589; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1590; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1591; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1592; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1593; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1594; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1595; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1596; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1597; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1598; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1599; SKIP-CACHE-INV-NEXT:    s_endpgm
1600;
1601; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg:
1602; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1603; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1604; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1605; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1606; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1607; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1608; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1609; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1610; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1611;
1612; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg:
1613; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1614; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1615; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1616; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1617; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1618; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1619; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1620; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1621; GFX90A-TGSPLIT-NEXT:    s_endpgm
1622    i32* %out, i32 %in, i32 %old) {
1623entry:
1624  %gep = getelementptr i32, i32* %out, i32 4
1625  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic
1626  ret void
1627}
1628
1629define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_cmpxchg(
1630; GFX7-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
1631; GFX7:       ; %bb.0: ; %entry
1632; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1633; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1634; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1635; GFX7-NEXT:    s_add_u32 s0, s0, 16
1636; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1637; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1638; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1639; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1640; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1641; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1642; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1643; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1644; GFX7-NEXT:    s_endpgm
1645;
1646; GFX10-WGP-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
1647; GFX10-WGP:       ; %bb.0: ; %entry
1648; GFX10-WGP-NEXT:    s_clause 0x1
1649; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1650; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1651; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1652; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1653; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1654; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1655; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1656; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1657; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1658; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1659; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1660; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1661; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1662; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1663; GFX10-WGP-NEXT:    buffer_gl0_inv
1664; GFX10-WGP-NEXT:    s_endpgm
1665;
1666; GFX10-CU-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
1667; GFX10-CU:       ; %bb.0: ; %entry
1668; GFX10-CU-NEXT:    s_clause 0x1
1669; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1670; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1671; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1672; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1673; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1674; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1675; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1676; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1677; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1678; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1679; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1680; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1681; GFX10-CU-NEXT:    s_endpgm
1682;
1683; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
1684; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1685; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1686; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1687; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1688; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1689; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1690; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1691; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1692; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1693; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1694; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1695; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1696; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1697; SKIP-CACHE-INV-NEXT:    s_endpgm
1698;
1699; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
1700; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1701; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1702; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1703; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1704; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1705; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1706; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1707; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1708; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1709; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1710;
1711; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
1712; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1713; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1714; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1715; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1716; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1717; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1718; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1719; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1720; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1721; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1722; GFX90A-TGSPLIT-NEXT:    s_endpgm
1723    i32* %out, i32 %in, i32 %old) {
1724entry:
1725  %gep = getelementptr i32, i32* %out, i32 4
1726  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
1727  ret void
1728}
1729
1730define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_cmpxchg(
1731; GFX7-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
1732; GFX7:       ; %bb.0: ; %entry
1733; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1734; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1735; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1736; GFX7-NEXT:    s_add_u32 s0, s0, 16
1737; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1738; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1739; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1740; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1741; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1742; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1743; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1744; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1745; GFX7-NEXT:    s_endpgm
1746;
1747; GFX10-WGP-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
1748; GFX10-WGP:       ; %bb.0: ; %entry
1749; GFX10-WGP-NEXT:    s_clause 0x1
1750; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1751; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1752; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1753; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1754; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1755; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1756; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1757; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1758; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1759; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1760; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1761; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1762; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1763; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1764; GFX10-WGP-NEXT:    buffer_gl0_inv
1765; GFX10-WGP-NEXT:    s_endpgm
1766;
1767; GFX10-CU-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
1768; GFX10-CU:       ; %bb.0: ; %entry
1769; GFX10-CU-NEXT:    s_clause 0x1
1770; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1771; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1772; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1773; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1774; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1775; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1776; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1777; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1778; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1779; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1780; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1781; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1782; GFX10-CU-NEXT:    s_endpgm
1783;
1784; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
1785; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1786; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1787; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1788; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1789; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1790; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1791; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1792; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1793; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1794; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1795; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1796; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1797; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1798; SKIP-CACHE-INV-NEXT:    s_endpgm
1799;
1800; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
1801; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1802; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1803; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1804; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1805; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1806; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1807; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1808; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1809; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1810; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1811;
1812; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
1813; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1814; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1815; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1816; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1817; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1818; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1819; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1820; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1821; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1822; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1823; GFX90A-TGSPLIT-NEXT:    s_endpgm
1824    i32* %out, i32 %in, i32 %old) {
1825entry:
1826  %gep = getelementptr i32, i32* %out, i32 4
1827  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
1828  ret void
1829}
1830
1831define amdgpu_kernel void @flat_workgroup_acquire_acquire_cmpxchg(
1832; GFX7-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
1833; GFX7:       ; %bb.0: ; %entry
1834; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1835; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1836; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1837; GFX7-NEXT:    s_add_u32 s0, s0, 16
1838; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1839; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1840; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1841; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1842; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1843; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1844; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1845; GFX7-NEXT:    s_endpgm
1846;
1847; GFX10-WGP-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
1848; GFX10-WGP:       ; %bb.0: ; %entry
1849; GFX10-WGP-NEXT:    s_clause 0x1
1850; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1851; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1852; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1853; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1854; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1855; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1856; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1857; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1858; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1859; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1860; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1861; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1862; GFX10-WGP-NEXT:    buffer_gl0_inv
1863; GFX10-WGP-NEXT:    s_endpgm
1864;
1865; GFX10-CU-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
1866; GFX10-CU:       ; %bb.0: ; %entry
1867; GFX10-CU-NEXT:    s_clause 0x1
1868; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1869; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1870; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1871; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1872; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1873; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1874; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1875; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1876; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1877; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1878; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1879; GFX10-CU-NEXT:    s_endpgm
1880;
1881; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
1882; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1883; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1884; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1885; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1886; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1887; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1888; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1889; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1890; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1891; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1892; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1893; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1894; SKIP-CACHE-INV-NEXT:    s_endpgm
1895;
1896; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
1897; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1898; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1899; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1900; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1901; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1902; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1903; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1904; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1905; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1906;
1907; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
1908; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1909; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1910; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1911; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1912; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1913; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1914; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1915; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1916; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1917; GFX90A-TGSPLIT-NEXT:    s_endpgm
1918    i32* %out, i32 %in, i32 %old) {
1919entry:
1920  %gep = getelementptr i32, i32* %out, i32 4
1921  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
1922  ret void
1923}
1924
1925define amdgpu_kernel void @flat_workgroup_release_acquire_cmpxchg(
1926; GFX7-LABEL: flat_workgroup_release_acquire_cmpxchg:
1927; GFX7:       ; %bb.0: ; %entry
1928; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1929; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1930; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1931; GFX7-NEXT:    s_add_u32 s0, s0, 16
1932; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1933; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1934; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1935; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1936; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1937; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1938; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1939; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1940; GFX7-NEXT:    s_endpgm
1941;
1942; GFX10-WGP-LABEL: flat_workgroup_release_acquire_cmpxchg:
1943; GFX10-WGP:       ; %bb.0: ; %entry
1944; GFX10-WGP-NEXT:    s_clause 0x1
1945; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1946; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1947; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1948; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1949; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1950; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1951; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1952; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1953; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1954; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1955; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1956; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1957; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1958; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1959; GFX10-WGP-NEXT:    buffer_gl0_inv
1960; GFX10-WGP-NEXT:    s_endpgm
1961;
1962; GFX10-CU-LABEL: flat_workgroup_release_acquire_cmpxchg:
1963; GFX10-CU:       ; %bb.0: ; %entry
1964; GFX10-CU-NEXT:    s_clause 0x1
1965; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1966; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1967; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1968; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1969; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1970; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1971; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1972; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1973; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1974; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1975; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1976; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1977; GFX10-CU-NEXT:    s_endpgm
1978;
1979; SKIP-CACHE-INV-LABEL: flat_workgroup_release_acquire_cmpxchg:
1980; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1981; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1982; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1983; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1984; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1985; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1986; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1987; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1988; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1989; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1990; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1991; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1992; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1993; SKIP-CACHE-INV-NEXT:    s_endpgm
1994;
1995; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg:
1996; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1997; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1998; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1999; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2000; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2001; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2002; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2003; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2004; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2005; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2006;
2007; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg:
2008; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2009; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2010; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2011; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2012; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2013; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2014; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2015; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2016; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2017; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2018; GFX90A-TGSPLIT-NEXT:    s_endpgm
2019    i32* %out, i32 %in, i32 %old) {
2020entry:
2021  %gep = getelementptr i32, i32* %out, i32 4
2022  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
2023  ret void
2024}
2025
2026define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_cmpxchg(
2027; GFX7-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
2028; GFX7:       ; %bb.0: ; %entry
2029; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2030; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2031; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2032; GFX7-NEXT:    s_add_u32 s0, s0, 16
2033; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2034; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2035; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2036; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2037; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2038; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2039; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2040; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2041; GFX7-NEXT:    s_endpgm
2042;
2043; GFX10-WGP-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
2044; GFX10-WGP:       ; %bb.0: ; %entry
2045; GFX10-WGP-NEXT:    s_clause 0x1
2046; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2047; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2048; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2049; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2050; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2051; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2052; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2053; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2054; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2055; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2056; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2057; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2058; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2059; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2060; GFX10-WGP-NEXT:    buffer_gl0_inv
2061; GFX10-WGP-NEXT:    s_endpgm
2062;
2063; GFX10-CU-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
2064; GFX10-CU:       ; %bb.0: ; %entry
2065; GFX10-CU-NEXT:    s_clause 0x1
2066; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2067; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2068; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2069; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2070; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2071; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2072; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2073; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2074; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2075; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2076; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2077; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2078; GFX10-CU-NEXT:    s_endpgm
2079;
2080; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
2081; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2082; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2083; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2084; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2085; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2086; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2087; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2088; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2089; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2090; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2091; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2092; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2093; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2094; SKIP-CACHE-INV-NEXT:    s_endpgm
2095;
2096; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
2097; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2098; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2099; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2100; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2101; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2102; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2103; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2104; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2105; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2106; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2107;
2108; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
2109; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2110; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2111; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2112; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2113; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2114; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2115; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2116; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2117; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2118; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2119; GFX90A-TGSPLIT-NEXT:    s_endpgm
2120    i32* %out, i32 %in, i32 %old) {
2121entry:
2122  %gep = getelementptr i32, i32* %out, i32 4
2123  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
2124  ret void
2125}
2126
2127define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_cmpxchg(
2128; GFX7-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
2129; GFX7:       ; %bb.0: ; %entry
2130; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2131; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2132; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2133; GFX7-NEXT:    s_add_u32 s0, s0, 16
2134; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2135; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2136; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2137; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2138; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2139; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2140; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2141; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2142; GFX7-NEXT:    s_endpgm
2143;
2144; GFX10-WGP-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
2145; GFX10-WGP:       ; %bb.0: ; %entry
2146; GFX10-WGP-NEXT:    s_clause 0x1
2147; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2148; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2149; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2150; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2151; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2152; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2153; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2154; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2155; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2156; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2157; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2158; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2159; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2160; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2161; GFX10-WGP-NEXT:    buffer_gl0_inv
2162; GFX10-WGP-NEXT:    s_endpgm
2163;
2164; GFX10-CU-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
2165; GFX10-CU:       ; %bb.0: ; %entry
2166; GFX10-CU-NEXT:    s_clause 0x1
2167; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2168; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2169; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2170; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2171; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2172; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2173; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2174; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2175; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2176; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2177; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2178; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2179; GFX10-CU-NEXT:    s_endpgm
2180;
2181; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
2182; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2183; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2184; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2185; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2186; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2187; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2188; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2189; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2190; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2191; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2192; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2193; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2194; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2195; SKIP-CACHE-INV-NEXT:    s_endpgm
2196;
2197; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
2198; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2199; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2200; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2201; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2202; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2203; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2204; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2205; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2206; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2207; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2208;
2209; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
2210; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2211; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2212; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2213; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2214; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2215; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2216; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2217; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2218; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2219; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2220; GFX90A-TGSPLIT-NEXT:    s_endpgm
2221    i32* %out, i32 %in, i32 %old) {
2222entry:
2223  %gep = getelementptr i32, i32* %out, i32 4
2224  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
2225  ret void
2226}
2227
2228define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_cmpxchg(
2229; GFX7-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
2230; GFX7:       ; %bb.0: ; %entry
2231; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2232; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2233; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2234; GFX7-NEXT:    s_add_u32 s0, s0, 16
2235; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2236; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2237; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2238; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2239; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2240; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2241; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2242; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2243; GFX7-NEXT:    s_endpgm
2244;
2245; GFX10-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
2246; GFX10-WGP:       ; %bb.0: ; %entry
2247; GFX10-WGP-NEXT:    s_clause 0x1
2248; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2249; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2250; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2251; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2252; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2253; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2254; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2255; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2256; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2257; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2258; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2259; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2260; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2261; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2262; GFX10-WGP-NEXT:    buffer_gl0_inv
2263; GFX10-WGP-NEXT:    s_endpgm
2264;
2265; GFX10-CU-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
2266; GFX10-CU:       ; %bb.0: ; %entry
2267; GFX10-CU-NEXT:    s_clause 0x1
2268; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2269; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2270; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2271; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2272; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2273; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2274; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2275; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2276; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2277; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2278; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2279; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2280; GFX10-CU-NEXT:    s_endpgm
2281;
2282; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
2283; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2284; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2285; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2286; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2287; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2288; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2289; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2290; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2291; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2292; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2293; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2294; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2295; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2296; SKIP-CACHE-INV-NEXT:    s_endpgm
2297;
2298; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
2299; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2300; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2301; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2302; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2303; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2304; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2305; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2306; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2307; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2308; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2309;
2310; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
2311; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2312; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2313; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2314; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2315; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2316; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2317; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2318; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2319; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2320; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2321; GFX90A-TGSPLIT-NEXT:    s_endpgm
2322    i32* %out, i32 %in, i32 %old) {
2323entry:
2324  %gep = getelementptr i32, i32* %out, i32 4
2325  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
2326  ret void
2327}
2328
2329define amdgpu_kernel void @flat_workgroup_acquire_monotonic_ret_cmpxchg(
2330; GFX7-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
2331; GFX7:       ; %bb.0: ; %entry
2332; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2333; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2334; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2335; GFX7-NEXT:    s_add_u32 s4, s0, 16
2336; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2337; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2338; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2339; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2340; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2341; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2342; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2343; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2344; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2345; GFX7-NEXT:    s_waitcnt vmcnt(0)
2346; GFX7-NEXT:    flat_store_dword v[0:1], v2
2347; GFX7-NEXT:    s_endpgm
2348;
2349; GFX10-WGP-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
2350; GFX10-WGP:       ; %bb.0: ; %entry
2351; GFX10-WGP-NEXT:    s_clause 0x1
2352; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2353; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2354; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2355; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2356; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2357; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2358; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2359; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2360; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2361; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2362; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2363; GFX10-WGP-NEXT:    buffer_gl0_inv
2364; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2365; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2366; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2367; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2368; GFX10-WGP-NEXT:    s_endpgm
2369;
2370; GFX10-CU-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
2371; GFX10-CU:       ; %bb.0: ; %entry
2372; GFX10-CU-NEXT:    s_clause 0x1
2373; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2374; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2375; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2376; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2377; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
2378; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
2379; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2380; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
2381; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2382; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2383; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2384; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2385; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2386; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
2387; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2388; GFX10-CU-NEXT:    s_endpgm
2389;
2390; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
2391; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2392; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2393; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2394; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2395; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
2396; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
2397; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2398; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2399; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2400; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2401; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2402; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2403; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2404; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2405; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2406; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2407; SKIP-CACHE-INV-NEXT:    s_endpgm
2408;
2409; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
2410; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2411; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2412; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2413; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2414; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2415; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2416; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2417; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2418; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2419; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2420;
2421; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
2422; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2423; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2424; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2425; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2426; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2427; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2428; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2429; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2430; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2431; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2432; GFX90A-TGSPLIT-NEXT:    s_endpgm
2433    i32* %out, i32 %in, i32 %old) {
2434entry:
2435  %gep = getelementptr i32, i32* %out, i32 4
2436  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
2437  %val0 = extractvalue { i32, i1 } %val, 0
2438  store i32 %val0, i32* %out, align 4
2439  ret void
2440}
2441
2442define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_ret_cmpxchg(
2443; GFX7-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
2444; GFX7:       ; %bb.0: ; %entry
2445; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2446; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2447; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2448; GFX7-NEXT:    s_add_u32 s4, s0, 16
2449; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2450; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2451; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2452; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2453; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2454; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2455; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2456; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2457; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2458; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2459; GFX7-NEXT:    s_waitcnt vmcnt(0)
2460; GFX7-NEXT:    flat_store_dword v[0:1], v2
2461; GFX7-NEXT:    s_endpgm
2462;
2463; GFX10-WGP-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
2464; GFX10-WGP:       ; %bb.0: ; %entry
2465; GFX10-WGP-NEXT:    s_clause 0x1
2466; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2467; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2468; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2469; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2470; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2471; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2472; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2473; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2474; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2475; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2476; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2477; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2478; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2479; GFX10-WGP-NEXT:    buffer_gl0_inv
2480; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2481; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2482; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2483; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2484; GFX10-WGP-NEXT:    s_endpgm
2485;
2486; GFX10-CU-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
2487; GFX10-CU:       ; %bb.0: ; %entry
2488; GFX10-CU-NEXT:    s_clause 0x1
2489; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2490; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2491; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2492; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2493; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
2494; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
2495; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2496; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
2497; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2498; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2499; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2500; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2501; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2502; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2503; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
2504; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2505; GFX10-CU-NEXT:    s_endpgm
2506;
2507; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
2508; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2509; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2510; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2511; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2512; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
2513; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
2514; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2515; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2516; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2517; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2518; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2519; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2520; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2521; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2522; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2523; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2524; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2525; SKIP-CACHE-INV-NEXT:    s_endpgm
2526;
2527; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
2528; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2529; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2530; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2531; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2532; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2533; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2534; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2535; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2536; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2537; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2538; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2539;
2540; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
2541; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2542; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2543; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2544; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2545; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2546; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2547; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2548; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2549; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2550; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2551; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2552; GFX90A-TGSPLIT-NEXT:    s_endpgm
2553    i32* %out, i32 %in, i32 %old) {
2554entry:
2555  %gep = getelementptr i32, i32* %out, i32 4
2556  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
2557  %val0 = extractvalue { i32, i1 } %val, 0
2558  store i32 %val0, i32* %out, align 4
2559  ret void
2560}
2561
2562define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_ret_cmpxchg(
2563; GFX7-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
2564; GFX7:       ; %bb.0: ; %entry
2565; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2566; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2567; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2568; GFX7-NEXT:    s_add_u32 s4, s0, 16
2569; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2570; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2571; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2572; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2573; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2574; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2575; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2576; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2577; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2578; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2579; GFX7-NEXT:    s_waitcnt vmcnt(0)
2580; GFX7-NEXT:    flat_store_dword v[0:1], v2
2581; GFX7-NEXT:    s_endpgm
2582;
2583; GFX10-WGP-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
2584; GFX10-WGP:       ; %bb.0: ; %entry
2585; GFX10-WGP-NEXT:    s_clause 0x1
2586; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2587; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2588; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2589; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2590; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2591; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2592; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2593; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2594; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2595; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2596; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2597; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2598; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2599; GFX10-WGP-NEXT:    buffer_gl0_inv
2600; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2601; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2602; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2603; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2604; GFX10-WGP-NEXT:    s_endpgm
2605;
2606; GFX10-CU-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
2607; GFX10-CU:       ; %bb.0: ; %entry
2608; GFX10-CU-NEXT:    s_clause 0x1
2609; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2610; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2611; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2612; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2613; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
2614; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
2615; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2616; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
2617; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2618; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2619; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2620; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2621; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2622; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2623; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
2624; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2625; GFX10-CU-NEXT:    s_endpgm
2626;
2627; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
2628; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2629; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2630; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2631; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2632; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
2633; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
2634; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2635; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2636; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2637; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2638; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2639; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2640; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2641; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2642; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2643; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2644; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2645; SKIP-CACHE-INV-NEXT:    s_endpgm
2646;
2647; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
2648; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2649; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2650; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2651; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2652; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2653; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2654; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2655; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2656; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2657; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2658; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2659;
2660; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
2661; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2662; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2663; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2664; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2665; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2666; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2667; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2668; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2669; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2670; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2671; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2672; GFX90A-TGSPLIT-NEXT:    s_endpgm
2673    i32* %out, i32 %in, i32 %old) {
2674entry:
2675  %gep = getelementptr i32, i32* %out, i32 4
2676  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
2677  %val0 = extractvalue { i32, i1 } %val, 0
2678  store i32 %val0, i32* %out, align 4
2679  ret void
2680}
2681
2682define amdgpu_kernel void @flat_workgroup_acquire_acquire_ret_cmpxchg(
2683; GFX7-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
2684; GFX7:       ; %bb.0: ; %entry
2685; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2686; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2687; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2688; GFX7-NEXT:    s_add_u32 s4, s0, 16
2689; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2690; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2691; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2692; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2693; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2694; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2695; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2696; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2697; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2698; GFX7-NEXT:    s_waitcnt vmcnt(0)
2699; GFX7-NEXT:    flat_store_dword v[0:1], v2
2700; GFX7-NEXT:    s_endpgm
2701;
2702; GFX10-WGP-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
2703; GFX10-WGP:       ; %bb.0: ; %entry
2704; GFX10-WGP-NEXT:    s_clause 0x1
2705; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2706; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2707; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2708; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2709; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2710; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2711; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2712; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2713; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2714; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2715; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2716; GFX10-WGP-NEXT:    buffer_gl0_inv
2717; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2718; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2719; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2720; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2721; GFX10-WGP-NEXT:    s_endpgm
2722;
2723; GFX10-CU-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
2724; GFX10-CU:       ; %bb.0: ; %entry
2725; GFX10-CU-NEXT:    s_clause 0x1
2726; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2727; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2728; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2729; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2730; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
2731; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
2732; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2733; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
2734; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2735; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2736; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2737; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2738; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2739; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
2740; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2741; GFX10-CU-NEXT:    s_endpgm
2742;
2743; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
2744; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2745; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2746; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2747; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2748; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
2749; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
2750; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2751; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2752; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2753; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2754; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2755; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2756; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2757; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2758; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2759; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2760; SKIP-CACHE-INV-NEXT:    s_endpgm
2761;
2762; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
2763; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2764; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2765; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2766; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2767; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2768; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2769; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2770; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2771; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2772; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2773;
2774; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
2775; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2776; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2777; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2778; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2779; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2780; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2781; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2782; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2783; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2784; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2785; GFX90A-TGSPLIT-NEXT:    s_endpgm
2786    i32* %out, i32 %in, i32 %old) {
2787entry:
2788  %gep = getelementptr i32, i32* %out, i32 4
2789  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
2790  %val0 = extractvalue { i32, i1 } %val, 0
2791  store i32 %val0, i32* %out, align 4
2792  ret void
2793}
2794
2795define amdgpu_kernel void @flat_workgroup_release_acquire_ret_cmpxchg(
2796; GFX7-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
2797; GFX7:       ; %bb.0: ; %entry
2798; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2799; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2800; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2801; GFX7-NEXT:    s_add_u32 s4, s0, 16
2802; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2803; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2804; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2805; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2806; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2807; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2808; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2809; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2810; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2811; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2812; GFX7-NEXT:    s_waitcnt vmcnt(0)
2813; GFX7-NEXT:    flat_store_dword v[0:1], v2
2814; GFX7-NEXT:    s_endpgm
2815;
2816; GFX10-WGP-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
2817; GFX10-WGP:       ; %bb.0: ; %entry
2818; GFX10-WGP-NEXT:    s_clause 0x1
2819; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2820; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2821; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2822; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2823; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2824; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2825; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2826; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2827; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2828; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2829; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2830; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2831; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2832; GFX10-WGP-NEXT:    buffer_gl0_inv
2833; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2834; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2835; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2836; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2837; GFX10-WGP-NEXT:    s_endpgm
2838;
2839; GFX10-CU-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
2840; GFX10-CU:       ; %bb.0: ; %entry
2841; GFX10-CU-NEXT:    s_clause 0x1
2842; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2843; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2844; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2845; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2846; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
2847; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
2848; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2849; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
2850; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2851; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2852; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2853; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2854; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2855; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2856; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
2857; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2858; GFX10-CU-NEXT:    s_endpgm
2859;
2860; SKIP-CACHE-INV-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
2861; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2862; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2863; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2864; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2865; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
2866; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
2867; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2868; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2869; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2870; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2871; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2872; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2873; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2874; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2875; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2876; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2877; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2878; SKIP-CACHE-INV-NEXT:    s_endpgm
2879;
2880; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
2881; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2882; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2883; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2884; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2885; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2886; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2887; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2888; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2889; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2890; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2891; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2892;
2893; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
2894; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2895; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2896; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2897; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2898; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2899; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2900; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2901; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2902; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2903; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2904; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2905; GFX90A-TGSPLIT-NEXT:    s_endpgm
2906    i32* %out, i32 %in, i32 %old) {
2907entry:
2908  %gep = getelementptr i32, i32* %out, i32 4
2909  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
2910  %val0 = extractvalue { i32, i1 } %val, 0
2911  store i32 %val0, i32* %out, align 4
2912  ret void
2913}
2914
2915define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_ret_cmpxchg(
2916; GFX7-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
2917; GFX7:       ; %bb.0: ; %entry
2918; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2919; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2920; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2921; GFX7-NEXT:    s_add_u32 s4, s0, 16
2922; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2923; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2924; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2925; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2926; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2927; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2928; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2929; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2930; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2931; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2932; GFX7-NEXT:    s_waitcnt vmcnt(0)
2933; GFX7-NEXT:    flat_store_dword v[0:1], v2
2934; GFX7-NEXT:    s_endpgm
2935;
2936; GFX10-WGP-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
2937; GFX10-WGP:       ; %bb.0: ; %entry
2938; GFX10-WGP-NEXT:    s_clause 0x1
2939; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2940; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2941; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2942; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2943; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2944; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2945; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2946; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2947; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2948; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2949; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2950; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2951; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2952; GFX10-WGP-NEXT:    buffer_gl0_inv
2953; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2954; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2955; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2956; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2957; GFX10-WGP-NEXT:    s_endpgm
2958;
2959; GFX10-CU-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
2960; GFX10-CU:       ; %bb.0: ; %entry
2961; GFX10-CU-NEXT:    s_clause 0x1
2962; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2963; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2964; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2965; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2966; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
2967; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
2968; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2969; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
2970; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2971; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2972; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2973; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2974; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2975; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2976; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
2977; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2978; GFX10-CU-NEXT:    s_endpgm
2979;
2980; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
2981; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2982; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2983; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2984; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2985; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
2986; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
2987; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2988; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2989; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2990; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2991; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2992; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2993; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2994; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2995; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2996; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2997; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2998; SKIP-CACHE-INV-NEXT:    s_endpgm
2999;
3000; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
3001; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3002; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3003; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3004; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3005; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3006; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3007; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3008; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3009; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3010; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3011; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3012;
3013; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
3014; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3015; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3016; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3017; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3018; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3019; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3020; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3021; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3022; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3023; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3024; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3025; GFX90A-TGSPLIT-NEXT:    s_endpgm
3026    i32* %out, i32 %in, i32 %old) {
3027entry:
3028  %gep = getelementptr i32, i32* %out, i32 4
3029  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
3030  %val0 = extractvalue { i32, i1 } %val, 0
3031  store i32 %val0, i32* %out, align 4
3032  ret void
3033}
3034
3035define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_ret_cmpxchg(
3036; GFX7-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
3037; GFX7:       ; %bb.0: ; %entry
3038; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3039; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3040; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3041; GFX7-NEXT:    s_add_u32 s4, s0, 16
3042; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3043; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3044; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3045; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3046; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3047; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3048; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3049; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3050; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3051; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3052; GFX7-NEXT:    s_waitcnt vmcnt(0)
3053; GFX7-NEXT:    flat_store_dword v[0:1], v2
3054; GFX7-NEXT:    s_endpgm
3055;
3056; GFX10-WGP-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
3057; GFX10-WGP:       ; %bb.0: ; %entry
3058; GFX10-WGP-NEXT:    s_clause 0x1
3059; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3060; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3061; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3062; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3063; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3064; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3065; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3066; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3067; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3068; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3069; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3070; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3071; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3072; GFX10-WGP-NEXT:    buffer_gl0_inv
3073; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3074; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3075; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3076; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3077; GFX10-WGP-NEXT:    s_endpgm
3078;
3079; GFX10-CU-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
3080; GFX10-CU:       ; %bb.0: ; %entry
3081; GFX10-CU-NEXT:    s_clause 0x1
3082; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3083; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3084; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3085; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3086; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3087; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3088; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3089; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3090; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3091; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3092; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3093; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3094; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3095; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3096; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3097; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3098; GFX10-CU-NEXT:    s_endpgm
3099;
3100; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
3101; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3102; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
3103; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3104; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3105; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3106; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3107; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3108; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3109; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3110; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3111; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3112; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3113; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3114; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3115; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3116; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3117; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3118; SKIP-CACHE-INV-NEXT:    s_endpgm
3119;
3120; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
3121; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3122; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3123; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3124; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3125; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3126; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3127; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3128; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3129; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3130; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3131; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3132;
3133; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
3134; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3135; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3136; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3137; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3138; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3139; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3140; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3141; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3142; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3143; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3144; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3145; GFX90A-TGSPLIT-NEXT:    s_endpgm
3146    i32* %out, i32 %in, i32 %old) {
3147entry:
3148  %gep = getelementptr i32, i32* %out, i32 4
3149  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
3150  %val0 = extractvalue { i32, i1 } %val, 0
3151  store i32 %val0, i32* %out, align 4
3152  ret void
3153}
3154
3155define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_ret_cmpxchg(
3156; GFX7-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
3157; GFX7:       ; %bb.0: ; %entry
3158; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3159; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3160; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3161; GFX7-NEXT:    s_add_u32 s4, s0, 16
3162; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3163; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3164; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3165; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3166; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3167; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3168; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3169; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3170; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3171; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3172; GFX7-NEXT:    s_waitcnt vmcnt(0)
3173; GFX7-NEXT:    flat_store_dword v[0:1], v2
3174; GFX7-NEXT:    s_endpgm
3175;
3176; GFX10-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
3177; GFX10-WGP:       ; %bb.0: ; %entry
3178; GFX10-WGP-NEXT:    s_clause 0x1
3179; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3180; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3181; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3182; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3183; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3184; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3185; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3186; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3187; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3188; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3189; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3190; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3191; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3192; GFX10-WGP-NEXT:    buffer_gl0_inv
3193; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3194; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3195; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3196; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3197; GFX10-WGP-NEXT:    s_endpgm
3198;
3199; GFX10-CU-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
3200; GFX10-CU:       ; %bb.0: ; %entry
3201; GFX10-CU-NEXT:    s_clause 0x1
3202; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3203; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3204; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3205; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3206; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3207; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3208; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3209; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3210; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3211; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3212; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3213; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3214; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3215; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3216; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3217; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3218; GFX10-CU-NEXT:    s_endpgm
3219;
3220; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
3221; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3222; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
3223; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3224; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3225; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3226; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3227; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3228; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3229; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3230; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3231; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3232; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3233; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3234; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3235; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3236; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3237; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3238; SKIP-CACHE-INV-NEXT:    s_endpgm
3239;
3240; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
3241; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3242; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3243; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3244; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3245; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3246; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3247; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3248; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3249; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3250; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3251; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3252;
3253; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
3254; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3255; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3256; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3257; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3258; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3259; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3260; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3261; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3262; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3263; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3264; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3265; GFX90A-TGSPLIT-NEXT:    s_endpgm
3266    i32* %out, i32 %in, i32 %old) {
3267entry:
3268  %gep = getelementptr i32, i32* %out, i32 4
3269  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
3270  %val0 = extractvalue { i32, i1 } %val, 0
3271  store i32 %val0, i32* %out, align 4
3272  ret void
3273}
3274
3275define amdgpu_kernel void @flat_workgroup_one_as_unordered_load(
3276; GFX7-LABEL: flat_workgroup_one_as_unordered_load:
3277; GFX7:       ; %bb.0: ; %entry
3278; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3279; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3280; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3281; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3282; GFX7-NEXT:    flat_load_dword v0, v[0:1]
3283; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3284; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3285; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3286; GFX7-NEXT:    flat_store_dword v[2:3], v0
3287; GFX7-NEXT:    s_endpgm
3288;
3289; GFX10-WGP-LABEL: flat_workgroup_one_as_unordered_load:
3290; GFX10-WGP:       ; %bb.0: ; %entry
3291; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3292; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3293; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3294; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3295; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
3296; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
3297; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
3298; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3299; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3300; GFX10-WGP-NEXT:    s_endpgm
3301;
3302; GFX10-CU-LABEL: flat_workgroup_one_as_unordered_load:
3303; GFX10-CU:       ; %bb.0: ; %entry
3304; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3305; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3306; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3307; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3308; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
3309; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
3310; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
3311; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3312; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3313; GFX10-CU-NEXT:    s_endpgm
3314;
3315; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_unordered_load:
3316; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3317; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
3318; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3319; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3320; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3321; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
3322; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
3323; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
3324; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3325; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
3326; SKIP-CACHE-INV-NEXT:    s_endpgm
3327;
3328; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_load:
3329; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3330; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3331; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3332; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3333; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3334; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1]
3335; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3336; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3337; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3338; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3339; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3340;
3341; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_load:
3342; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3343; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3344; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3345; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3346; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3347; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1]
3348; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3349; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3350; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3351; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3352; GFX90A-TGSPLIT-NEXT:    s_endpgm
3353    i32* %in, i32* %out) {
3354entry:
3355  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") unordered, align 4
3356  store i32 %val, i32* %out
3357  ret void
3358}
3359
3360define amdgpu_kernel void @flat_workgroup_one_as_monotonic_load(
3361; GFX7-LABEL: flat_workgroup_one_as_monotonic_load:
3362; GFX7:       ; %bb.0: ; %entry
3363; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3364; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3365; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3366; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3367; GFX7-NEXT:    flat_load_dword v0, v[0:1]
3368; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3369; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3370; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3371; GFX7-NEXT:    flat_store_dword v[2:3], v0
3372; GFX7-NEXT:    s_endpgm
3373;
3374; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_load:
3375; GFX10-WGP:       ; %bb.0: ; %entry
3376; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3377; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3378; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3379; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3380; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
3381; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
3382; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
3383; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3384; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3385; GFX10-WGP-NEXT:    s_endpgm
3386;
3387; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_load:
3388; GFX10-CU:       ; %bb.0: ; %entry
3389; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3390; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3391; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3392; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3393; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
3394; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
3395; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
3396; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3397; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3398; GFX10-CU-NEXT:    s_endpgm
3399;
3400; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_load:
3401; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3402; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
3403; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3404; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3405; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3406; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
3407; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
3408; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
3409; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3410; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
3411; SKIP-CACHE-INV-NEXT:    s_endpgm
3412;
3413; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load:
3414; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3415; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3416; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3417; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3418; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3419; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1]
3420; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3421; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3422; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3423; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3424; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3425;
3426; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load:
3427; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3428; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3429; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3430; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3431; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3432; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc
3433; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3434; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3435; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3436; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3437; GFX90A-TGSPLIT-NEXT:    s_endpgm
3438    i32* %in, i32* %out) {
3439entry:
3440  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") monotonic, align 4
3441  store i32 %val, i32* %out
3442  ret void
3443}
3444
3445define amdgpu_kernel void @flat_workgroup_one_as_acquire_load(
3446; GFX7-LABEL: flat_workgroup_one_as_acquire_load:
3447; GFX7:       ; %bb.0: ; %entry
3448; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3449; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3450; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3451; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3452; GFX7-NEXT:    flat_load_dword v0, v[0:1]
3453; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3454; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3455; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3456; GFX7-NEXT:    flat_store_dword v[2:3], v0
3457; GFX7-NEXT:    s_endpgm
3458;
3459; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_load:
3460; GFX10-WGP:       ; %bb.0: ; %entry
3461; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3462; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3463; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3464; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3465; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
3466; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3467; GFX10-WGP-NEXT:    buffer_gl0_inv
3468; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
3469; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
3470; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3471; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3472; GFX10-WGP-NEXT:    s_endpgm
3473;
3474; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_load:
3475; GFX10-CU:       ; %bb.0: ; %entry
3476; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3477; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3478; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3479; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3480; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
3481; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
3482; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
3483; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3484; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3485; GFX10-CU-NEXT:    s_endpgm
3486;
3487; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_load:
3488; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3489; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
3490; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3491; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3492; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3493; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
3494; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
3495; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
3496; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3497; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
3498; SKIP-CACHE-INV-NEXT:    s_endpgm
3499;
3500; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_load:
3501; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3502; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3503; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3504; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3505; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3506; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1]
3507; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3508; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3509; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3510; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3511; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3512;
3513; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_load:
3514; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3515; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3516; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3517; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3518; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3519; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc
3520; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3521; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3522; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3523; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3524; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3525; GFX90A-TGSPLIT-NEXT:    s_endpgm
3526    i32* %in, i32* %out) {
3527entry:
3528  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") acquire, align 4
3529  store i32 %val, i32* %out
3530  ret void
3531}
3532
3533define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_load(
3534; GFX7-LABEL: flat_workgroup_one_as_seq_cst_load:
3535; GFX7:       ; %bb.0: ; %entry
3536; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3537; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3538; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3539; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3540; GFX7-NEXT:    flat_load_dword v0, v[0:1]
3541; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3542; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3543; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3544; GFX7-NEXT:    flat_store_dword v[2:3], v0
3545; GFX7-NEXT:    s_endpgm
3546;
3547; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_load:
3548; GFX10-WGP:       ; %bb.0: ; %entry
3549; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3550; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3551; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3552; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3553; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3554; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3555; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
3556; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3557; GFX10-WGP-NEXT:    buffer_gl0_inv
3558; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
3559; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
3560; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3561; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3562; GFX10-WGP-NEXT:    s_endpgm
3563;
3564; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_load:
3565; GFX10-CU:       ; %bb.0: ; %entry
3566; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3567; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3568; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3569; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3570; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
3571; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
3572; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
3573; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3574; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3575; GFX10-CU-NEXT:    s_endpgm
3576;
3577; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_load:
3578; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3579; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
3580; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3581; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3582; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3583; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
3584; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
3585; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
3586; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3587; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
3588; SKIP-CACHE-INV-NEXT:    s_endpgm
3589;
3590; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load:
3591; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3592; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3593; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3594; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3595; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3596; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1]
3597; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3598; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3599; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3600; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3601; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3602;
3603; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load:
3604; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3605; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3606; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3607; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3608; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3609; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3610; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc
3611; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3612; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3613; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3614; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3615; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3616; GFX90A-TGSPLIT-NEXT:    s_endpgm
3617    i32* %in, i32* %out) {
3618entry:
3619  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") seq_cst, align 4
3620  store i32 %val, i32* %out
3621  ret void
3622}
3623
3624define amdgpu_kernel void @flat_workgroup_one_as_unordered_store(
3625; GFX7-LABEL: flat_workgroup_one_as_unordered_store:
3626; GFX7:       ; %bb.0: ; %entry
3627; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
3628; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
3629; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3630; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3631; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3632; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3633; GFX7-NEXT:    flat_store_dword v[0:1], v2
3634; GFX7-NEXT:    s_endpgm
3635;
3636; GFX10-WGP-LABEL: flat_workgroup_one_as_unordered_store:
3637; GFX10-WGP:       ; %bb.0: ; %entry
3638; GFX10-WGP-NEXT:    s_clause 0x1
3639; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3640; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
3641; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3642; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3643; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3644; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3645; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3646; GFX10-WGP-NEXT:    s_endpgm
3647;
3648; GFX10-CU-LABEL: flat_workgroup_one_as_unordered_store:
3649; GFX10-CU:       ; %bb.0: ; %entry
3650; GFX10-CU-NEXT:    s_clause 0x1
3651; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3652; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
3653; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3654; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3655; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3656; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3657; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3658; GFX10-CU-NEXT:    s_endpgm
3659;
3660; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_unordered_store:
3661; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3662; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
3663; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3664; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3665; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
3666; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3667; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3668; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3669; SKIP-CACHE-INV-NEXT:    s_endpgm
3670;
3671; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_store:
3672; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3673; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3674; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3675; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3676; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3677; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3678; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3679; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3680;
3681; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_store:
3682; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3683; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3684; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3685; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3686; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3687; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3688; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3689; GFX90A-TGSPLIT-NEXT:    s_endpgm
3690    i32 %in, i32* %out) {
3691entry:
3692  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") unordered, align 4
3693  ret void
3694}
3695
3696define amdgpu_kernel void @flat_workgroup_one_as_monotonic_store(
3697; GFX7-LABEL: flat_workgroup_one_as_monotonic_store:
3698; GFX7:       ; %bb.0: ; %entry
3699; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
3700; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
3701; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3702; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3703; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3704; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3705; GFX7-NEXT:    flat_store_dword v[0:1], v2
3706; GFX7-NEXT:    s_endpgm
3707;
3708; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_store:
3709; GFX10-WGP:       ; %bb.0: ; %entry
3710; GFX10-WGP-NEXT:    s_clause 0x1
3711; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3712; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
3713; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3714; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3715; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3716; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3717; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3718; GFX10-WGP-NEXT:    s_endpgm
3719;
3720; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_store:
3721; GFX10-CU:       ; %bb.0: ; %entry
3722; GFX10-CU-NEXT:    s_clause 0x1
3723; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3724; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
3725; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3726; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3727; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3728; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3729; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3730; GFX10-CU-NEXT:    s_endpgm
3731;
3732; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_store:
3733; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3734; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
3735; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3736; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3737; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
3738; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3739; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3740; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3741; SKIP-CACHE-INV-NEXT:    s_endpgm
3742;
3743; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store:
3744; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3745; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3746; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3747; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3748; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3749; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3750; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3751; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3752;
3753; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store:
3754; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3755; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3756; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3757; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3758; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3759; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3760; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3761; GFX90A-TGSPLIT-NEXT:    s_endpgm
3762    i32 %in, i32* %out) {
3763entry:
3764  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") monotonic, align 4
3765  ret void
3766}
3767
3768define amdgpu_kernel void @flat_workgroup_one_as_release_store(
3769; GFX7-LABEL: flat_workgroup_one_as_release_store:
3770; GFX7:       ; %bb.0: ; %entry
3771; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
3772; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
3773; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3774; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3775; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3776; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3777; GFX7-NEXT:    flat_store_dword v[0:1], v2
3778; GFX7-NEXT:    s_endpgm
3779;
3780; GFX10-WGP-LABEL: flat_workgroup_one_as_release_store:
3781; GFX10-WGP:       ; %bb.0: ; %entry
3782; GFX10-WGP-NEXT:    s_clause 0x1
3783; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3784; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
3785; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3786; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3787; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3788; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3789; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3790; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3791; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3792; GFX10-WGP-NEXT:    s_endpgm
3793;
3794; GFX10-CU-LABEL: flat_workgroup_one_as_release_store:
3795; GFX10-CU:       ; %bb.0: ; %entry
3796; GFX10-CU-NEXT:    s_clause 0x1
3797; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3798; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
3799; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3800; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3801; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3802; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3803; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3804; GFX10-CU-NEXT:    s_endpgm
3805;
3806; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_store:
3807; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3808; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
3809; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3810; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3811; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
3812; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3813; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3814; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3815; SKIP-CACHE-INV-NEXT:    s_endpgm
3816;
3817; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_store:
3818; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3819; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3820; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3821; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3822; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3823; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3824; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3825; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3826;
3827; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_store:
3828; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3829; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3830; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3831; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3832; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3833; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3834; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3835; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3836; GFX90A-TGSPLIT-NEXT:    s_endpgm
3837    i32 %in, i32* %out) {
3838entry:
3839  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") release, align 4
3840  ret void
3841}
3842
3843define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_store(
3844; GFX7-LABEL: flat_workgroup_one_as_seq_cst_store:
3845; GFX7:       ; %bb.0: ; %entry
3846; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
3847; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
3848; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3849; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3850; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3851; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3852; GFX7-NEXT:    flat_store_dword v[0:1], v2
3853; GFX7-NEXT:    s_endpgm
3854;
3855; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_store:
3856; GFX10-WGP:       ; %bb.0: ; %entry
3857; GFX10-WGP-NEXT:    s_clause 0x1
3858; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3859; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
3860; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3861; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3862; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3863; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3864; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3865; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3866; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3867; GFX10-WGP-NEXT:    s_endpgm
3868;
3869; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_store:
3870; GFX10-CU:       ; %bb.0: ; %entry
3871; GFX10-CU-NEXT:    s_clause 0x1
3872; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3873; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
3874; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3875; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3876; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3877; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3878; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3879; GFX10-CU-NEXT:    s_endpgm
3880;
3881; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_store:
3882; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3883; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
3884; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3885; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3886; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
3887; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3888; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3889; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3890; SKIP-CACHE-INV-NEXT:    s_endpgm
3891;
3892; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store:
3893; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3894; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3895; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3896; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3897; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3898; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3899; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3900; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3901;
3902; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store:
3903; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3904; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3905; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3906; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3907; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3908; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3909; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3910; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3911; GFX90A-TGSPLIT-NEXT:    s_endpgm
3912    i32 %in, i32* %out) {
3913entry:
3914  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") seq_cst, align 4
3915  ret void
3916}
3917
3918define amdgpu_kernel void @flat_workgroup_one_as_monotonic_atomicrmw(
3919; GFX7-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
3920; GFX7:       ; %bb.0: ; %entry
3921; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3922; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
3923; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3924; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3925; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3926; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3927; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
3928; GFX7-NEXT:    s_endpgm
3929;
3930; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
3931; GFX10-WGP:       ; %bb.0: ; %entry
3932; GFX10-WGP-NEXT:    s_clause 0x1
3933; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3934; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
3935; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3936; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3937; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3938; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3939; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
3940; GFX10-WGP-NEXT:    s_endpgm
3941;
3942; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
3943; GFX10-CU:       ; %bb.0: ; %entry
3944; GFX10-CU-NEXT:    s_clause 0x1
3945; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3946; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
3947; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3948; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3949; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3950; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3951; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
3952; GFX10-CU-NEXT:    s_endpgm
3953;
3954; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
3955; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3956; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
3957; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
3958; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3959; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3960; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3961; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3962; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
3963; SKIP-CACHE-INV-NEXT:    s_endpgm
3964;
3965; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
3966; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3967; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3968; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
3969; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3970; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3971; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3972; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
3973; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3974;
3975; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
3976; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3977; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3978; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
3979; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3980; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3981; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3982; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
3983; GFX90A-TGSPLIT-NEXT:    s_endpgm
3984    i32* %out, i32 %in) {
3985entry:
3986  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") monotonic
3987  ret void
3988}
3989
3990define amdgpu_kernel void @flat_workgroup_one_as_acquire_atomicrmw(
3991; GFX7-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
3992; GFX7:       ; %bb.0: ; %entry
3993; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3994; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
3995; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3996; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3997; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3998; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3999; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4000; GFX7-NEXT:    s_endpgm
4001;
4002; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
4003; GFX10-WGP:       ; %bb.0: ; %entry
4004; GFX10-WGP-NEXT:    s_clause 0x1
4005; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4006; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4007; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4008; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4009; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4010; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4011; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
4012; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4013; GFX10-WGP-NEXT:    buffer_gl0_inv
4014; GFX10-WGP-NEXT:    s_endpgm
4015;
4016; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
4017; GFX10-CU:       ; %bb.0: ; %entry
4018; GFX10-CU-NEXT:    s_clause 0x1
4019; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4020; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4021; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4022; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4023; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4024; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4025; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
4026; GFX10-CU-NEXT:    s_endpgm
4027;
4028; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
4029; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4030; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4031; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4032; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4033; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4034; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4035; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4036; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
4037; SKIP-CACHE-INV-NEXT:    s_endpgm
4038;
4039; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
4040; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4041; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4042; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4043; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4044; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4045; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4046; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4047; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4048;
4049; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
4050; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4051; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4052; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4053; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4054; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4055; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4056; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4057; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4058; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4059; GFX90A-TGSPLIT-NEXT:    s_endpgm
4060    i32* %out, i32 %in) {
4061entry:
4062  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acquire
4063  ret void
4064}
4065
4066define amdgpu_kernel void @flat_workgroup_one_as_release_atomicrmw(
4067; GFX7-LABEL: flat_workgroup_one_as_release_atomicrmw:
4068; GFX7:       ; %bb.0: ; %entry
4069; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4070; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4071; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4072; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4073; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4074; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4075; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4076; GFX7-NEXT:    s_endpgm
4077;
4078; GFX10-WGP-LABEL: flat_workgroup_one_as_release_atomicrmw:
4079; GFX10-WGP:       ; %bb.0: ; %entry
4080; GFX10-WGP-NEXT:    s_clause 0x1
4081; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4082; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4083; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4084; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4085; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4086; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4087; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4088; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4089; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
4090; GFX10-WGP-NEXT:    s_endpgm
4091;
4092; GFX10-CU-LABEL: flat_workgroup_one_as_release_atomicrmw:
4093; GFX10-CU:       ; %bb.0: ; %entry
4094; GFX10-CU-NEXT:    s_clause 0x1
4095; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4096; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4097; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4098; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4099; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4100; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4101; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
4102; GFX10-CU-NEXT:    s_endpgm
4103;
4104; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_atomicrmw:
4105; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4106; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4107; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4108; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4109; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4110; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4111; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4112; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
4113; SKIP-CACHE-INV-NEXT:    s_endpgm
4114;
4115; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw:
4116; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4117; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4118; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4119; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4120; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4121; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4122; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4123; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4124;
4125; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw:
4126; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4127; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4128; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4129; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4130; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4131; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4132; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4133; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4134; GFX90A-TGSPLIT-NEXT:    s_endpgm
4135    i32* %out, i32 %in) {
4136entry:
4137  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") release
4138  ret void
4139}
4140
4141define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_atomicrmw(
4142; GFX7-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
4143; GFX7:       ; %bb.0: ; %entry
4144; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4145; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4146; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4147; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4148; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4149; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4150; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4151; GFX7-NEXT:    s_endpgm
4152;
4153; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
4154; GFX10-WGP:       ; %bb.0: ; %entry
4155; GFX10-WGP-NEXT:    s_clause 0x1
4156; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4157; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4158; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4159; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4160; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4161; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4162; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4163; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4164; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
4165; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4166; GFX10-WGP-NEXT:    buffer_gl0_inv
4167; GFX10-WGP-NEXT:    s_endpgm
4168;
4169; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
4170; GFX10-CU:       ; %bb.0: ; %entry
4171; GFX10-CU-NEXT:    s_clause 0x1
4172; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4173; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4174; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4175; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4176; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4177; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4178; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
4179; GFX10-CU-NEXT:    s_endpgm
4180;
4181; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
4182; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4183; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4184; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4185; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4186; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4187; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4188; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4189; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
4190; SKIP-CACHE-INV-NEXT:    s_endpgm
4191;
4192; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
4193; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4194; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4195; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4196; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4197; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4198; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4199; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4200; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4201;
4202; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
4203; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4204; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4205; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4206; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4207; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4208; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4209; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4210; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4211; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4212; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4213; GFX90A-TGSPLIT-NEXT:    s_endpgm
4214    i32* %out, i32 %in) {
4215entry:
4216  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acq_rel
4217  ret void
4218}
4219
4220define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_atomicrmw(
4221; GFX7-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
4222; GFX7:       ; %bb.0: ; %entry
4223; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4224; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4225; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4226; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4227; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4228; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4229; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4230; GFX7-NEXT:    s_endpgm
4231;
4232; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
4233; GFX10-WGP:       ; %bb.0: ; %entry
4234; GFX10-WGP-NEXT:    s_clause 0x1
4235; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4236; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4237; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4238; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4239; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4240; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4241; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4242; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4243; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
4244; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4245; GFX10-WGP-NEXT:    buffer_gl0_inv
4246; GFX10-WGP-NEXT:    s_endpgm
4247;
4248; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
4249; GFX10-CU:       ; %bb.0: ; %entry
4250; GFX10-CU-NEXT:    s_clause 0x1
4251; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4252; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4253; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4254; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4255; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4256; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4257; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
4258; GFX10-CU-NEXT:    s_endpgm
4259;
4260; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
4261; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4262; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4263; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4264; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4265; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4266; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4267; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4268; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
4269; SKIP-CACHE-INV-NEXT:    s_endpgm
4270;
4271; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
4272; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4273; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4274; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4275; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4276; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4277; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4278; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4279; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4280;
4281; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
4282; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4283; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4284; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4285; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4286; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4287; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4288; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4289; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4290; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4291; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4292; GFX90A-TGSPLIT-NEXT:    s_endpgm
4293    i32* %out, i32 %in) {
4294entry:
4295  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") seq_cst
4296  ret void
4297}
4298
4299define amdgpu_kernel void @flat_workgroup_one_as_acquire_ret_atomicrmw(
4300; GFX7-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
4301; GFX7:       ; %bb.0: ; %entry
4302; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4303; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4304; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4305; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4306; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4307; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4308; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4309; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4310; GFX7-NEXT:    flat_store_dword v[0:1], v2
4311; GFX7-NEXT:    s_endpgm
4312;
4313; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
4314; GFX10-WGP:       ; %bb.0: ; %entry
4315; GFX10-WGP-NEXT:    s_clause 0x1
4316; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4317; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4318; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4319; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4320; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4321; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4322; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4323; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4324; GFX10-WGP-NEXT:    buffer_gl0_inv
4325; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4326; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4327; GFX10-WGP-NEXT:    s_endpgm
4328;
4329; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
4330; GFX10-CU:       ; %bb.0: ; %entry
4331; GFX10-CU-NEXT:    s_clause 0x1
4332; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4333; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4334; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4335; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4336; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4337; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4338; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4339; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4340; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4341; GFX10-CU-NEXT:    s_endpgm
4342;
4343; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
4344; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4345; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4346; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4347; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4348; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4349; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4350; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4351; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4352; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4353; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4354; SKIP-CACHE-INV-NEXT:    s_endpgm
4355;
4356; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
4357; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4358; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4359; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4360; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4361; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4362; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4363; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4364; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4365; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4366; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4367;
4368; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
4369; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4370; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4371; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4372; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4373; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4374; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4375; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4376; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4377; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4378; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4379; GFX90A-TGSPLIT-NEXT:    s_endpgm
4380    i32* %out, i32 %in) {
4381entry:
4382  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acquire
4383  store i32 %val, i32* %out, align 4
4384  ret void
4385}
4386
4387define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_ret_atomicrmw(
4388; GFX7-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
4389; GFX7:       ; %bb.0: ; %entry
4390; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4391; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4392; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4393; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4394; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4395; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4396; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4397; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4398; GFX7-NEXT:    flat_store_dword v[0:1], v2
4399; GFX7-NEXT:    s_endpgm
4400;
4401; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
4402; GFX10-WGP:       ; %bb.0: ; %entry
4403; GFX10-WGP-NEXT:    s_clause 0x1
4404; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4405; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4406; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4407; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4408; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4409; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4410; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4411; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4412; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4413; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4414; GFX10-WGP-NEXT:    buffer_gl0_inv
4415; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4416; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4417; GFX10-WGP-NEXT:    s_endpgm
4418;
4419; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
4420; GFX10-CU:       ; %bb.0: ; %entry
4421; GFX10-CU-NEXT:    s_clause 0x1
4422; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4423; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4424; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4425; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4426; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4427; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4428; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4429; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4430; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4431; GFX10-CU-NEXT:    s_endpgm
4432;
4433; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
4434; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4435; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4436; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4437; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4438; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4439; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4440; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4441; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4442; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4443; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4444; SKIP-CACHE-INV-NEXT:    s_endpgm
4445;
4446; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
4447; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4448; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4449; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4450; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4451; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4452; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4453; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4454; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4455; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4456; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4457;
4458; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
4459; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4460; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4461; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4462; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4463; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4464; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4465; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4466; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4467; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4468; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4469; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4470; GFX90A-TGSPLIT-NEXT:    s_endpgm
4471    i32* %out, i32 %in) {
4472entry:
4473  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acq_rel
4474  store i32 %val, i32* %out, align 4
4475  ret void
4476}
4477
4478define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_ret_atomicrmw(
4479; GFX7-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
4480; GFX7:       ; %bb.0: ; %entry
4481; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4482; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4483; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4484; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4485; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4486; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4487; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4488; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4489; GFX7-NEXT:    flat_store_dword v[0:1], v2
4490; GFX7-NEXT:    s_endpgm
4491;
4492; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
4493; GFX10-WGP:       ; %bb.0: ; %entry
4494; GFX10-WGP-NEXT:    s_clause 0x1
4495; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4496; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4497; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4498; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4499; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4500; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4501; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4502; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4503; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4504; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4505; GFX10-WGP-NEXT:    buffer_gl0_inv
4506; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4507; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4508; GFX10-WGP-NEXT:    s_endpgm
4509;
4510; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
4511; GFX10-CU:       ; %bb.0: ; %entry
4512; GFX10-CU-NEXT:    s_clause 0x1
4513; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4514; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4515; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4516; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4517; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4518; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4519; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4520; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4521; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4522; GFX10-CU-NEXT:    s_endpgm
4523;
4524; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
4525; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4526; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4527; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4528; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4529; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4530; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4531; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4532; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4533; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4534; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4535; SKIP-CACHE-INV-NEXT:    s_endpgm
4536;
4537; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
4538; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4539; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4540; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4541; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4542; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4543; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4544; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4545; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4546; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4547; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4548;
4549; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
4550; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4551; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4552; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4553; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4554; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4555; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4556; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4557; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4558; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4559; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4560; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4561; GFX90A-TGSPLIT-NEXT:    s_endpgm
4562    i32* %out, i32 %in) {
4563entry:
4564  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") seq_cst
4565  store i32 %val, i32* %out, align 4
4566  ret void
4567}
4568
4569define amdgpu_kernel void @flat_workgroup_one_as_monotonic_monotonic_cmpxchg(
4570; GFX7-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
4571; GFX7:       ; %bb.0: ; %entry
4572; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4573; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4574; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4575; GFX7-NEXT:    s_add_u32 s0, s0, 16
4576; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4577; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4578; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4579; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4580; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4581; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4582; GFX7-NEXT:    s_endpgm
4583;
4584; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
4585; GFX10-WGP:       ; %bb.0: ; %entry
4586; GFX10-WGP-NEXT:    s_clause 0x1
4587; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4588; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4589; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4590; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4591; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4592; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4593; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4594; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4595; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4596; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4597; GFX10-WGP-NEXT:    s_endpgm
4598;
4599; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
4600; GFX10-CU:       ; %bb.0: ; %entry
4601; GFX10-CU-NEXT:    s_clause 0x1
4602; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4603; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4604; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4605; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4606; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4607; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4608; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4609; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4610; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4611; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4612; GFX10-CU-NEXT:    s_endpgm
4613;
4614; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
4615; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4616; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4617; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
4618; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4619; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4620; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4621; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4622; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4623; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4624; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4625; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4626; SKIP-CACHE-INV-NEXT:    s_endpgm
4627;
4628; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
4629; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4630; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4631; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4632; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4633; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4634; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4635; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4636; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4637;
4638; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
4639; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4640; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4641; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4642; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4643; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4644; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4645; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4646; GFX90A-TGSPLIT-NEXT:    s_endpgm
4647    i32* %out, i32 %in, i32 %old) {
4648entry:
4649  %gep = getelementptr i32, i32* %out, i32 4
4650  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic
4651  ret void
4652}
4653
4654define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_cmpxchg(
4655; GFX7-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
4656; GFX7:       ; %bb.0: ; %entry
4657; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4658; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4659; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4660; GFX7-NEXT:    s_add_u32 s0, s0, 16
4661; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4662; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4663; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4664; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4665; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4666; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4667; GFX7-NEXT:    s_endpgm
4668;
4669; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
4670; GFX10-WGP:       ; %bb.0: ; %entry
4671; GFX10-WGP-NEXT:    s_clause 0x1
4672; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4673; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4674; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4675; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4676; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4677; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4678; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4679; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4680; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4681; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4682; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4683; GFX10-WGP-NEXT:    buffer_gl0_inv
4684; GFX10-WGP-NEXT:    s_endpgm
4685;
4686; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
4687; GFX10-CU:       ; %bb.0: ; %entry
4688; GFX10-CU-NEXT:    s_clause 0x1
4689; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4690; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4691; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4692; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4693; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4694; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4695; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4696; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4697; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4698; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4699; GFX10-CU-NEXT:    s_endpgm
4700;
4701; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
4702; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4703; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4704; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
4705; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4706; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4707; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4708; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4709; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4710; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4711; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4712; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4713; SKIP-CACHE-INV-NEXT:    s_endpgm
4714;
4715; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
4716; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4717; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4718; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4719; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4720; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4721; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4722; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4723; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4724;
4725; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
4726; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4727; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4728; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4729; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4730; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4731; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4732; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4733; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4734; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4735; GFX90A-TGSPLIT-NEXT:    s_endpgm
4736    i32* %out, i32 %in, i32 %old) {
4737entry:
4738  %gep = getelementptr i32, i32* %out, i32 4
4739  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
4740  ret void
4741}
4742
4743define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_cmpxchg(
4744; GFX7-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
4745; GFX7:       ; %bb.0: ; %entry
4746; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4747; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4748; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4749; GFX7-NEXT:    s_add_u32 s0, s0, 16
4750; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4751; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4752; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4753; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4754; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4755; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4756; GFX7-NEXT:    s_endpgm
4757;
4758; GFX10-WGP-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
4759; GFX10-WGP:       ; %bb.0: ; %entry
4760; GFX10-WGP-NEXT:    s_clause 0x1
4761; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4762; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4763; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4764; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4765; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4766; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4767; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4768; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4769; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4770; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4771; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4772; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4773; GFX10-WGP-NEXT:    s_endpgm
4774;
4775; GFX10-CU-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
4776; GFX10-CU:       ; %bb.0: ; %entry
4777; GFX10-CU-NEXT:    s_clause 0x1
4778; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4779; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4780; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4781; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4782; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4783; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4784; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4785; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4786; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4787; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4788; GFX10-CU-NEXT:    s_endpgm
4789;
4790; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
4791; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4792; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4793; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
4794; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4795; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4796; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4797; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4798; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4799; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4800; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4801; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4802; SKIP-CACHE-INV-NEXT:    s_endpgm
4803;
4804; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
4805; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4806; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4807; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4808; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4809; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4810; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4811; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4812; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4813;
4814; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
4815; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4816; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4817; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4818; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4819; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4820; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4821; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4822; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4823; GFX90A-TGSPLIT-NEXT:    s_endpgm
4824    i32* %out, i32 %in, i32 %old) {
4825entry:
4826  %gep = getelementptr i32, i32* %out, i32 4
4827  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic
4828  ret void
4829}
4830
4831define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_cmpxchg(
4832; GFX7-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
4833; GFX7:       ; %bb.0: ; %entry
4834; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4835; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4836; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4837; GFX7-NEXT:    s_add_u32 s0, s0, 16
4838; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4839; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4840; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4841; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4842; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4843; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4844; GFX7-NEXT:    s_endpgm
4845;
4846; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
4847; GFX10-WGP:       ; %bb.0: ; %entry
4848; GFX10-WGP-NEXT:    s_clause 0x1
4849; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4850; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4851; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4852; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4853; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4854; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4855; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4856; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4857; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4858; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4859; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4860; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4861; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4862; GFX10-WGP-NEXT:    buffer_gl0_inv
4863; GFX10-WGP-NEXT:    s_endpgm
4864;
4865; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
4866; GFX10-CU:       ; %bb.0: ; %entry
4867; GFX10-CU-NEXT:    s_clause 0x1
4868; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4869; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4870; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4871; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4872; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4873; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4874; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4875; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4876; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4877; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4878; GFX10-CU-NEXT:    s_endpgm
4879;
4880; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
4881; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4882; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4883; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
4884; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4885; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4886; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4887; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4888; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4889; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4890; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4891; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4892; SKIP-CACHE-INV-NEXT:    s_endpgm
4893;
4894; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
4895; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4896; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4897; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4898; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4899; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4900; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4901; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4902; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4903;
4904; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
4905; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4906; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4907; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4908; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4909; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4910; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4911; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4912; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4913; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4914; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4915; GFX90A-TGSPLIT-NEXT:    s_endpgm
4916    i32* %out, i32 %in, i32 %old) {
4917entry:
4918  %gep = getelementptr i32, i32* %out, i32 4
4919  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
4920  ret void
4921}
4922
4923define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_cmpxchg(
4924; GFX7-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
4925; GFX7:       ; %bb.0: ; %entry
4926; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4927; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4928; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4929; GFX7-NEXT:    s_add_u32 s0, s0, 16
4930; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4931; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4932; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4933; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4934; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4935; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4936; GFX7-NEXT:    s_endpgm
4937;
4938; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
4939; GFX10-WGP:       ; %bb.0: ; %entry
4940; GFX10-WGP-NEXT:    s_clause 0x1
4941; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4942; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4943; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4944; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4945; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4946; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4947; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4948; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4949; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4950; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4951; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4952; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4953; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4954; GFX10-WGP-NEXT:    buffer_gl0_inv
4955; GFX10-WGP-NEXT:    s_endpgm
4956;
4957; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
4958; GFX10-CU:       ; %bb.0: ; %entry
4959; GFX10-CU-NEXT:    s_clause 0x1
4960; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4961; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4962; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4963; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4964; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4965; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4966; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4967; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4968; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4969; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4970; GFX10-CU-NEXT:    s_endpgm
4971;
4972; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
4973; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4974; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4975; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
4976; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4977; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4978; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4979; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4980; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4981; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4982; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4983; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4984; SKIP-CACHE-INV-NEXT:    s_endpgm
4985;
4986; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
4987; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4988; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4989; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4990; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4991; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4992; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4993; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4994; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4995;
4996; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
4997; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4998; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4999; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5000; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5001; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5002; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5003; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5004; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5005; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5006; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5007; GFX90A-TGSPLIT-NEXT:    s_endpgm
5008    i32* %out, i32 %in, i32 %old) {
5009entry:
5010  %gep = getelementptr i32, i32* %out, i32 4
5011  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
5012  ret void
5013}
5014
5015define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_cmpxchg(
5016; GFX7-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
5017; GFX7:       ; %bb.0: ; %entry
5018; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5019; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5020; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5021; GFX7-NEXT:    s_add_u32 s0, s0, 16
5022; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5023; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5024; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5025; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5026; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5027; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5028; GFX7-NEXT:    s_endpgm
5029;
5030; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
5031; GFX10-WGP:       ; %bb.0: ; %entry
5032; GFX10-WGP-NEXT:    s_clause 0x1
5033; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5034; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5035; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5036; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5037; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5038; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5039; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5040; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5041; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5042; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5043; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5044; GFX10-WGP-NEXT:    buffer_gl0_inv
5045; GFX10-WGP-NEXT:    s_endpgm
5046;
5047; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
5048; GFX10-CU:       ; %bb.0: ; %entry
5049; GFX10-CU-NEXT:    s_clause 0x1
5050; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5051; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5052; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5053; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5054; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5055; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5056; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5057; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5058; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5059; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5060; GFX10-CU-NEXT:    s_endpgm
5061;
5062; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
5063; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5064; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5065; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5066; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5067; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5068; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5069; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5070; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5071; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5072; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5073; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5074; SKIP-CACHE-INV-NEXT:    s_endpgm
5075;
5076; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
5077; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5078; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5079; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5080; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5081; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5082; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5083; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5084; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5085;
5086; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
5087; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5088; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5089; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5090; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5091; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5092; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5093; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5094; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5095; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5096; GFX90A-TGSPLIT-NEXT:    s_endpgm
5097    i32* %out, i32 %in, i32 %old) {
5098entry:
5099  %gep = getelementptr i32, i32* %out, i32 4
5100  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
5101  ret void
5102}
5103
5104define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_cmpxchg(
5105; GFX7-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
5106; GFX7:       ; %bb.0: ; %entry
5107; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5108; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5109; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5110; GFX7-NEXT:    s_add_u32 s0, s0, 16
5111; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5112; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5113; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5114; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5115; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5116; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5117; GFX7-NEXT:    s_endpgm
5118;
5119; GFX10-WGP-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
5120; GFX10-WGP:       ; %bb.0: ; %entry
5121; GFX10-WGP-NEXT:    s_clause 0x1
5122; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5123; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5124; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5125; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5126; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5127; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5128; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5129; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5130; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5131; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5132; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5133; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5134; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5135; GFX10-WGP-NEXT:    buffer_gl0_inv
5136; GFX10-WGP-NEXT:    s_endpgm
5137;
5138; GFX10-CU-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
5139; GFX10-CU:       ; %bb.0: ; %entry
5140; GFX10-CU-NEXT:    s_clause 0x1
5141; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5142; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5143; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5144; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5145; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5146; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5147; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5148; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5149; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5150; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5151; GFX10-CU-NEXT:    s_endpgm
5152;
5153; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
5154; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5155; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5156; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5157; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5158; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5159; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5160; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5161; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5162; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5163; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5164; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5165; SKIP-CACHE-INV-NEXT:    s_endpgm
5166;
5167; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
5168; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5169; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5170; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5171; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5172; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5173; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5174; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5175; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5176;
5177; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
5178; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5179; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5180; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5181; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5182; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5183; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5184; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5185; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5186; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5187; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5188; GFX90A-TGSPLIT-NEXT:    s_endpgm
5189    i32* %out, i32 %in, i32 %old) {
5190entry:
5191  %gep = getelementptr i32, i32* %out, i32 4
5192  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
5193  ret void
5194}
5195
5196define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_cmpxchg(
5197; GFX7-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
5198; GFX7:       ; %bb.0: ; %entry
5199; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5200; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5201; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5202; GFX7-NEXT:    s_add_u32 s0, s0, 16
5203; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5204; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5205; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5206; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5207; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5208; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5209; GFX7-NEXT:    s_endpgm
5210;
5211; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
5212; GFX10-WGP:       ; %bb.0: ; %entry
5213; GFX10-WGP-NEXT:    s_clause 0x1
5214; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5215; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5216; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5217; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5218; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5219; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5220; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5221; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5222; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5223; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5224; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5225; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5226; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5227; GFX10-WGP-NEXT:    buffer_gl0_inv
5228; GFX10-WGP-NEXT:    s_endpgm
5229;
5230; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
5231; GFX10-CU:       ; %bb.0: ; %entry
5232; GFX10-CU-NEXT:    s_clause 0x1
5233; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5234; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5235; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5236; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5237; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5238; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5239; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5240; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5241; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5242; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5243; GFX10-CU-NEXT:    s_endpgm
5244;
5245; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
5246; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5247; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5248; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5249; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5250; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5251; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5252; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5253; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5254; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5255; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5256; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5257; SKIP-CACHE-INV-NEXT:    s_endpgm
5258;
5259; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
5260; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5261; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5262; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5263; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5264; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5265; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5266; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5267; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5268;
5269; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
5270; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5271; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5272; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5273; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5274; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5275; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5276; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5277; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5278; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5279; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5280; GFX90A-TGSPLIT-NEXT:    s_endpgm
5281    i32* %out, i32 %in, i32 %old) {
5282entry:
5283  %gep = getelementptr i32, i32* %out, i32 4
5284  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
5285  ret void
5286}
5287
5288define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_cmpxchg(
5289; GFX7-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
5290; GFX7:       ; %bb.0: ; %entry
5291; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5292; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5293; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5294; GFX7-NEXT:    s_add_u32 s0, s0, 16
5295; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5296; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5297; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5298; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5299; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5300; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5301; GFX7-NEXT:    s_endpgm
5302;
5303; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
5304; GFX10-WGP:       ; %bb.0: ; %entry
5305; GFX10-WGP-NEXT:    s_clause 0x1
5306; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5307; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5308; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5309; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5310; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5311; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5312; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5313; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5314; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5315; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5316; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5317; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5318; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5319; GFX10-WGP-NEXT:    buffer_gl0_inv
5320; GFX10-WGP-NEXT:    s_endpgm
5321;
5322; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
5323; GFX10-CU:       ; %bb.0: ; %entry
5324; GFX10-CU-NEXT:    s_clause 0x1
5325; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5326; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5327; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5328; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5329; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5330; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5331; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5332; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5333; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5334; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5335; GFX10-CU-NEXT:    s_endpgm
5336;
5337; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
5338; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5339; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5340; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5341; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5342; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5343; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5344; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5345; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5346; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5347; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5348; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5349; SKIP-CACHE-INV-NEXT:    s_endpgm
5350;
5351; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
5352; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5353; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5354; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5355; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5356; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5357; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5358; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5359; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5360;
5361; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
5362; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5363; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5364; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5365; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5366; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5367; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5368; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5369; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5370; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5371; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5372; GFX90A-TGSPLIT-NEXT:    s_endpgm
5373    i32* %out, i32 %in, i32 %old) {
5374entry:
5375  %gep = getelementptr i32, i32* %out, i32 4
5376  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
5377  ret void
5378}
5379
5380define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
5381; GFX7-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
5382; GFX7:       ; %bb.0: ; %entry
5383; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5384; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5385; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5386; GFX7-NEXT:    s_add_u32 s0, s0, 16
5387; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5388; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5389; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5390; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5391; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5392; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5393; GFX7-NEXT:    s_endpgm
5394;
5395; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
5396; GFX10-WGP:       ; %bb.0: ; %entry
5397; GFX10-WGP-NEXT:    s_clause 0x1
5398; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5399; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5400; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5401; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5402; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5403; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5404; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5405; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5406; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5407; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5408; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5409; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5410; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5411; GFX10-WGP-NEXT:    buffer_gl0_inv
5412; GFX10-WGP-NEXT:    s_endpgm
5413;
5414; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
5415; GFX10-CU:       ; %bb.0: ; %entry
5416; GFX10-CU-NEXT:    s_clause 0x1
5417; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5418; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5419; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5420; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5421; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5422; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5423; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5424; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5425; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5426; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5427; GFX10-CU-NEXT:    s_endpgm
5428;
5429; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
5430; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5431; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5432; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5433; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5434; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5435; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5436; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5437; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5438; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5439; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5440; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5441; SKIP-CACHE-INV-NEXT:    s_endpgm
5442;
5443; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
5444; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5445; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5446; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5447; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5448; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5449; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5450; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5451; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5452;
5453; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
5454; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5455; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5456; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5457; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5458; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5459; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5460; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5461; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5462; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5463; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5464; GFX90A-TGSPLIT-NEXT:    s_endpgm
5465    i32* %out, i32 %in, i32 %old) {
5466entry:
5467  %gep = getelementptr i32, i32* %out, i32 4
5468  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
5469  ret void
5470}
5471
5472define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
5473; GFX7-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
5474; GFX7:       ; %bb.0: ; %entry
5475; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5476; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5477; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5478; GFX7-NEXT:    s_add_u32 s4, s0, 16
5479; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5480; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5481; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5482; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5483; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5484; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5485; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5486; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5487; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5488; GFX7-NEXT:    flat_store_dword v[0:1], v2
5489; GFX7-NEXT:    s_endpgm
5490;
5491; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
5492; GFX10-WGP:       ; %bb.0: ; %entry
5493; GFX10-WGP-NEXT:    s_clause 0x1
5494; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5495; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5496; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5497; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5498; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5499; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5500; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5501; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5502; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5503; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5504; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5505; GFX10-WGP-NEXT:    buffer_gl0_inv
5506; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5507; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5508; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5509; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5510; GFX10-WGP-NEXT:    s_endpgm
5511;
5512; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
5513; GFX10-CU:       ; %bb.0: ; %entry
5514; GFX10-CU-NEXT:    s_clause 0x1
5515; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5516; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5517; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5518; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5519; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5520; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5521; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5522; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5523; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5524; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5525; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5526; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5527; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5528; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5529; GFX10-CU-NEXT:    s_endpgm
5530;
5531; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
5532; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5533; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5534; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5535; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5536; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5537; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5538; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5539; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5540; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5541; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5542; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5543; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5544; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5545; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5546; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5547; SKIP-CACHE-INV-NEXT:    s_endpgm
5548;
5549; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
5550; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5551; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5552; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5553; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5554; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5555; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5556; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5557; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5558; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5559; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5560;
5561; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
5562; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5563; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5564; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5565; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5566; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5567; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5568; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5569; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5570; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5571; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5572; GFX90A-TGSPLIT-NEXT:    s_endpgm
5573    i32* %out, i32 %in, i32 %old) {
5574entry:
5575  %gep = getelementptr i32, i32* %out, i32 4
5576  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
5577  %val0 = extractvalue { i32, i1 } %val, 0
5578  store i32 %val0, i32* %out, align 4
5579  ret void
5580}
5581
5582define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
5583; GFX7-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
5584; GFX7:       ; %bb.0: ; %entry
5585; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5586; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5587; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5588; GFX7-NEXT:    s_add_u32 s4, s0, 16
5589; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5590; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5591; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5592; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5593; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5594; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5595; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5596; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5597; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5598; GFX7-NEXT:    flat_store_dword v[0:1], v2
5599; GFX7-NEXT:    s_endpgm
5600;
5601; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
5602; GFX10-WGP:       ; %bb.0: ; %entry
5603; GFX10-WGP-NEXT:    s_clause 0x1
5604; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5605; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5606; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5607; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5608; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5609; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5610; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5611; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5612; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5613; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5614; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5615; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5616; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5617; GFX10-WGP-NEXT:    buffer_gl0_inv
5618; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5619; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5620; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5621; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5622; GFX10-WGP-NEXT:    s_endpgm
5623;
5624; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
5625; GFX10-CU:       ; %bb.0: ; %entry
5626; GFX10-CU-NEXT:    s_clause 0x1
5627; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5628; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5629; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5630; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5631; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5632; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5633; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5634; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5635; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5636; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5637; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5638; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5639; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5640; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5641; GFX10-CU-NEXT:    s_endpgm
5642;
5643; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
5644; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5645; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5646; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5647; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5648; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5649; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5650; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5651; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5652; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5653; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5654; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5655; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5656; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5657; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5658; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5659; SKIP-CACHE-INV-NEXT:    s_endpgm
5660;
5661; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
5662; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5663; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5664; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5665; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5666; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5667; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5668; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5669; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5670; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5671; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5672;
5673; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
5674; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5675; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5676; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5677; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5678; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5679; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5680; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5681; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5682; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5683; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5684; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5685; GFX90A-TGSPLIT-NEXT:    s_endpgm
5686    i32* %out, i32 %in, i32 %old) {
5687entry:
5688  %gep = getelementptr i32, i32* %out, i32 4
5689  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
5690  %val0 = extractvalue { i32, i1 } %val, 0
5691  store i32 %val0, i32* %out, align 4
5692  ret void
5693}
5694
5695define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
5696; GFX7-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
5697; GFX7:       ; %bb.0: ; %entry
5698; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5699; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5700; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5701; GFX7-NEXT:    s_add_u32 s4, s0, 16
5702; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5703; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5704; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5705; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5706; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5707; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5708; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5709; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5710; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5711; GFX7-NEXT:    flat_store_dword v[0:1], v2
5712; GFX7-NEXT:    s_endpgm
5713;
5714; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
5715; GFX10-WGP:       ; %bb.0: ; %entry
5716; GFX10-WGP-NEXT:    s_clause 0x1
5717; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5718; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5719; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5720; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5721; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5722; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5723; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5724; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5725; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5726; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5727; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5728; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5729; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5730; GFX10-WGP-NEXT:    buffer_gl0_inv
5731; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5732; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5733; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5734; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5735; GFX10-WGP-NEXT:    s_endpgm
5736;
5737; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
5738; GFX10-CU:       ; %bb.0: ; %entry
5739; GFX10-CU-NEXT:    s_clause 0x1
5740; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5741; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5742; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5743; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5744; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5745; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5746; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5747; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5748; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5749; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5750; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5751; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5752; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5753; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5754; GFX10-CU-NEXT:    s_endpgm
5755;
5756; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
5757; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5758; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5759; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5760; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5761; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5762; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5763; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5764; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5765; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5766; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5767; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5768; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5769; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5770; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5771; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5772; SKIP-CACHE-INV-NEXT:    s_endpgm
5773;
5774; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
5775; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5776; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5777; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5778; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5779; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5780; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5781; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5782; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5783; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5784; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5785;
5786; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
5787; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5788; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5789; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5790; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5791; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5792; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5793; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5794; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5795; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5796; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5797; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5798; GFX90A-TGSPLIT-NEXT:    s_endpgm
5799    i32* %out, i32 %in, i32 %old) {
5800entry:
5801  %gep = getelementptr i32, i32* %out, i32 4
5802  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
5803  %val0 = extractvalue { i32, i1 } %val, 0
5804  store i32 %val0, i32* %out, align 4
5805  ret void
5806}
5807
5808define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_ret_cmpxchg(
5809; GFX7-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
5810; GFX7:       ; %bb.0: ; %entry
5811; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5812; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5813; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5814; GFX7-NEXT:    s_add_u32 s4, s0, 16
5815; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5816; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5817; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5818; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5819; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5820; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5821; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5822; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5823; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5824; GFX7-NEXT:    flat_store_dword v[0:1], v2
5825; GFX7-NEXT:    s_endpgm
5826;
5827; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
5828; GFX10-WGP:       ; %bb.0: ; %entry
5829; GFX10-WGP-NEXT:    s_clause 0x1
5830; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5831; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5832; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5833; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5834; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5835; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5836; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5837; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5838; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5839; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5840; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5841; GFX10-WGP-NEXT:    buffer_gl0_inv
5842; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5843; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5844; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5845; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5846; GFX10-WGP-NEXT:    s_endpgm
5847;
5848; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
5849; GFX10-CU:       ; %bb.0: ; %entry
5850; GFX10-CU-NEXT:    s_clause 0x1
5851; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5852; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5853; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5854; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5855; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5856; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5857; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5858; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5859; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5860; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5861; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5862; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5863; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5864; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5865; GFX10-CU-NEXT:    s_endpgm
5866;
5867; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
5868; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5869; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5870; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5871; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5872; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5873; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5874; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5875; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5876; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5877; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5878; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5879; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5880; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5881; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5882; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5883; SKIP-CACHE-INV-NEXT:    s_endpgm
5884;
5885; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
5886; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5887; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5888; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5889; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5890; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5891; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5892; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5893; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5894; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5895; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5896;
5897; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
5898; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5899; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5900; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5901; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5902; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5903; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5904; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5905; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5906; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5907; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5908; GFX90A-TGSPLIT-NEXT:    s_endpgm
5909    i32* %out, i32 %in, i32 %old) {
5910entry:
5911  %gep = getelementptr i32, i32* %out, i32 4
5912  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
5913  %val0 = extractvalue { i32, i1 } %val, 0
5914  store i32 %val0, i32* %out, align 4
5915  ret void
5916}
5917
5918define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_ret_cmpxchg(
5919; GFX7-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
5920; GFX7:       ; %bb.0: ; %entry
5921; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5922; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5923; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5924; GFX7-NEXT:    s_add_u32 s4, s0, 16
5925; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5926; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5927; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5928; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5929; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5930; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5931; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5932; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5933; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5934; GFX7-NEXT:    flat_store_dword v[0:1], v2
5935; GFX7-NEXT:    s_endpgm
5936;
5937; GFX10-WGP-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
5938; GFX10-WGP:       ; %bb.0: ; %entry
5939; GFX10-WGP-NEXT:    s_clause 0x1
5940; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5941; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5942; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5943; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5944; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5945; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5946; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5947; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5948; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5949; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5950; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5951; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5952; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5953; GFX10-WGP-NEXT:    buffer_gl0_inv
5954; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5955; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5956; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5957; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5958; GFX10-WGP-NEXT:    s_endpgm
5959;
5960; GFX10-CU-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
5961; GFX10-CU:       ; %bb.0: ; %entry
5962; GFX10-CU-NEXT:    s_clause 0x1
5963; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5964; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5965; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5966; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5967; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5968; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5969; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5970; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5971; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5972; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5973; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5974; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5975; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5976; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5977; GFX10-CU-NEXT:    s_endpgm
5978;
5979; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
5980; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5981; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5982; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5983; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5984; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5985; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5986; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5987; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5988; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5989; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5990; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5991; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5992; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5993; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5994; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5995; SKIP-CACHE-INV-NEXT:    s_endpgm
5996;
5997; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
5998; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5999; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6000; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6001; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6002; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6003; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6004; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6005; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6006; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6007; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6008;
6009; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
6010; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6011; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6012; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6013; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6014; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6015; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6016; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6017; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6018; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6019; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6020; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6021; GFX90A-TGSPLIT-NEXT:    s_endpgm
6022    i32* %out, i32 %in, i32 %old) {
6023entry:
6024  %gep = getelementptr i32, i32* %out, i32 4
6025  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
6026  %val0 = extractvalue { i32, i1 } %val, 0
6027  store i32 %val0, i32* %out, align 4
6028  ret void
6029}
6030
6031define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
6032; GFX7-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
6033; GFX7:       ; %bb.0: ; %entry
6034; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6035; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6036; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6037; GFX7-NEXT:    s_add_u32 s4, s0, 16
6038; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6039; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6040; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6041; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6042; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6043; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6044; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6045; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6046; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6047; GFX7-NEXT:    flat_store_dword v[0:1], v2
6048; GFX7-NEXT:    s_endpgm
6049;
6050; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
6051; GFX10-WGP:       ; %bb.0: ; %entry
6052; GFX10-WGP-NEXT:    s_clause 0x1
6053; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6054; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6055; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6056; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6057; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6058; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6059; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6060; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6061; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6062; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6063; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6064; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6065; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6066; GFX10-WGP-NEXT:    buffer_gl0_inv
6067; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6068; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6069; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6070; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6071; GFX10-WGP-NEXT:    s_endpgm
6072;
6073; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
6074; GFX10-CU:       ; %bb.0: ; %entry
6075; GFX10-CU-NEXT:    s_clause 0x1
6076; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6077; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6078; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6079; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6080; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6081; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6082; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6083; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6084; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6085; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6086; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6087; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6088; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6089; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6090; GFX10-CU-NEXT:    s_endpgm
6091;
6092; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
6093; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6094; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
6095; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6096; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6097; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6098; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6099; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6100; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6101; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6102; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6103; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6104; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6105; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6106; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6107; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6108; SKIP-CACHE-INV-NEXT:    s_endpgm
6109;
6110; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
6111; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6112; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6113; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6114; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6115; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6116; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6117; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6118; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6119; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6120; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6121;
6122; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
6123; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6124; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6125; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6126; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6127; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6128; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6129; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6130; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6131; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6132; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6133; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6134; GFX90A-TGSPLIT-NEXT:    s_endpgm
6135    i32* %out, i32 %in, i32 %old) {
6136entry:
6137  %gep = getelementptr i32, i32* %out, i32 4
6138  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
6139  %val0 = extractvalue { i32, i1 } %val, 0
6140  store i32 %val0, i32* %out, align 4
6141  ret void
6142}
6143
6144define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
6145; GFX7-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
6146; GFX7:       ; %bb.0: ; %entry
6147; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6148; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6149; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6150; GFX7-NEXT:    s_add_u32 s4, s0, 16
6151; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6152; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6153; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6154; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6155; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6156; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6157; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6158; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6159; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6160; GFX7-NEXT:    flat_store_dword v[0:1], v2
6161; GFX7-NEXT:    s_endpgm
6162;
6163; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
6164; GFX10-WGP:       ; %bb.0: ; %entry
6165; GFX10-WGP-NEXT:    s_clause 0x1
6166; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6167; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6168; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6169; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6170; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6171; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6172; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6173; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6174; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6175; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6176; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6177; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6178; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6179; GFX10-WGP-NEXT:    buffer_gl0_inv
6180; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6181; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6182; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6183; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6184; GFX10-WGP-NEXT:    s_endpgm
6185;
6186; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
6187; GFX10-CU:       ; %bb.0: ; %entry
6188; GFX10-CU-NEXT:    s_clause 0x1
6189; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6190; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6191; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6192; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6193; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6194; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6195; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6196; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6197; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6198; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6199; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6200; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6201; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6202; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6203; GFX10-CU-NEXT:    s_endpgm
6204;
6205; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
6206; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6207; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
6208; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6209; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6210; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6211; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6212; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6213; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6214; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6215; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6216; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6217; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6218; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6219; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6220; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6221; SKIP-CACHE-INV-NEXT:    s_endpgm
6222;
6223; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
6224; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6225; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6226; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6227; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6228; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6229; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6230; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6231; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6232; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6233; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6234;
6235; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
6236; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6237; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6238; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6239; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6240; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6241; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6242; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6243; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6244; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6245; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6246; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6247; GFX90A-TGSPLIT-NEXT:    s_endpgm
6248    i32* %out, i32 %in, i32 %old) {
6249entry:
6250  %gep = getelementptr i32, i32* %out, i32 4
6251  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
6252  %val0 = extractvalue { i32, i1 } %val, 0
6253  store i32 %val0, i32* %out, align 4
6254  ret void
6255}
6256
6257define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
6258; GFX7-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
6259; GFX7:       ; %bb.0: ; %entry
6260; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6261; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6262; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6263; GFX7-NEXT:    s_add_u32 s4, s0, 16
6264; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6265; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6266; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6267; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6268; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6269; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6270; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6271; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6272; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6273; GFX7-NEXT:    flat_store_dword v[0:1], v2
6274; GFX7-NEXT:    s_endpgm
6275;
6276; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
6277; GFX10-WGP:       ; %bb.0: ; %entry
6278; GFX10-WGP-NEXT:    s_clause 0x1
6279; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6280; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6281; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6282; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6283; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6284; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6285; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6286; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6287; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6288; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6289; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6290; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6291; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6292; GFX10-WGP-NEXT:    buffer_gl0_inv
6293; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6294; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6295; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6296; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6297; GFX10-WGP-NEXT:    s_endpgm
6298;
6299; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
6300; GFX10-CU:       ; %bb.0: ; %entry
6301; GFX10-CU-NEXT:    s_clause 0x1
6302; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6303; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6304; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6305; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6306; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6307; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6308; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6309; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6310; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6311; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6312; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6313; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6314; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6315; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6316; GFX10-CU-NEXT:    s_endpgm
6317;
6318; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
6319; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6320; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
6321; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6322; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6323; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6324; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6325; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6326; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6327; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6328; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6329; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6330; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6331; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6332; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6333; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6334; SKIP-CACHE-INV-NEXT:    s_endpgm
6335;
6336; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
6337; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6338; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6339; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6340; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6341; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6342; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6343; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6344; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6345; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6346; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6347;
6348; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
6349; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6350; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6351; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6352; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6353; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6354; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6355; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6356; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6357; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6358; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6359; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6360; GFX90A-TGSPLIT-NEXT:    s_endpgm
6361    i32* %out, i32 %in, i32 %old) {
6362entry:
6363  %gep = getelementptr i32, i32* %out, i32 4
6364  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
6365  %val0 = extractvalue { i32, i1 } %val, 0
6366  store i32 %val0, i32* %out, align 4
6367  ret void
6368}
6369
6370