1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
8
9define amdgpu_kernel void @flat_workgroup_unordered_load(
10; GFX7-LABEL: flat_workgroup_unordered_load:
11; GFX7:       ; %bb.0: ; %entry
12; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
13; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14; GFX7-NEXT:    v_mov_b32_e32 v0, s0
15; GFX7-NEXT:    v_mov_b32_e32 v1, s1
16; GFX7-NEXT:    flat_load_dword v2, v[0:1]
17; GFX7-NEXT:    v_mov_b32_e32 v0, s2
18; GFX7-NEXT:    v_mov_b32_e32 v1, s3
19; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
20; GFX7-NEXT:    flat_store_dword v[0:1], v2
21; GFX7-NEXT:    s_endpgm
22;
23; GFX10-WGP-LABEL: flat_workgroup_unordered_load:
24; GFX10-WGP:       ; %bb.0: ; %entry
25; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
26; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
27; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
28; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
29; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
30; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
31; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
32; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
33; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
34; GFX10-WGP-NEXT:    s_endpgm
35;
36; GFX10-CU-LABEL: flat_workgroup_unordered_load:
37; GFX10-CU:       ; %bb.0: ; %entry
38; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
39; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
40; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
41; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
42; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
43; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
44; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
45; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
46; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
47; GFX10-CU-NEXT:    s_endpgm
48;
49; SKIP-CACHE-INV-LABEL: flat_workgroup_unordered_load:
50; SKIP-CACHE-INV:       ; %bb.0: ; %entry
51; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
52; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
53; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
54; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
55; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
56; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
57; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
58; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
59; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
60; SKIP-CACHE-INV-NEXT:    s_endpgm
61;
62; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_unordered_load:
63; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
64; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
65; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
66; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
67; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
68; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
69; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
70; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
71; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
72; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
73; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
74;
75; GFX90A-TGSPLIT-LABEL: flat_workgroup_unordered_load:
76; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
77; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
78; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
79; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
80; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
81; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
82; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
83; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
84; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
85; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
86; GFX90A-TGSPLIT-NEXT:    s_endpgm
87    i32* %in, i32* %out) {
88entry:
89  %val = load atomic i32, i32* %in syncscope("workgroup") unordered, align 4
90  store i32 %val, i32* %out
91  ret void
92}
93
94define amdgpu_kernel void @flat_workgroup_monotonic_load(
95; GFX7-LABEL: flat_workgroup_monotonic_load:
96; GFX7:       ; %bb.0: ; %entry
97; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
98; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
99; GFX7-NEXT:    v_mov_b32_e32 v0, s0
100; GFX7-NEXT:    v_mov_b32_e32 v1, s1
101; GFX7-NEXT:    flat_load_dword v2, v[0:1]
102; GFX7-NEXT:    v_mov_b32_e32 v0, s2
103; GFX7-NEXT:    v_mov_b32_e32 v1, s3
104; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
105; GFX7-NEXT:    flat_store_dword v[0:1], v2
106; GFX7-NEXT:    s_endpgm
107;
108; GFX10-WGP-LABEL: flat_workgroup_monotonic_load:
109; GFX10-WGP:       ; %bb.0: ; %entry
110; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
111; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
112; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
113; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
114; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
115; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
116; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
117; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
118; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
119; GFX10-WGP-NEXT:    s_endpgm
120;
121; GFX10-CU-LABEL: flat_workgroup_monotonic_load:
122; GFX10-CU:       ; %bb.0: ; %entry
123; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
124; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
125; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
126; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
127; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
128; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
129; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
130; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
131; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
132; GFX10-CU-NEXT:    s_endpgm
133;
134; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_load:
135; SKIP-CACHE-INV:       ; %bb.0: ; %entry
136; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
137; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
138; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
139; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
140; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
141; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
142; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
143; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
144; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
145; SKIP-CACHE-INV-NEXT:    s_endpgm
146;
147; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_load:
148; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
149; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
150; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
151; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
152; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
153; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
154; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
155; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
156; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
157; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
158; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
159;
160; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_load:
161; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
162; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
163; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
164; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
165; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
166; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
167; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
168; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
169; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
170; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
171; GFX90A-TGSPLIT-NEXT:    s_endpgm
172    i32* %in, i32* %out) {
173entry:
174  %val = load atomic i32, i32* %in syncscope("workgroup") monotonic, align 4
175  store i32 %val, i32* %out
176  ret void
177}
178
179define amdgpu_kernel void @flat_workgroup_acquire_load(
180; GFX7-LABEL: flat_workgroup_acquire_load:
181; GFX7:       ; %bb.0: ; %entry
182; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
183; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
184; GFX7-NEXT:    v_mov_b32_e32 v0, s0
185; GFX7-NEXT:    v_mov_b32_e32 v1, s1
186; GFX7-NEXT:    flat_load_dword v2, v[0:1]
187; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
188; GFX7-NEXT:    v_mov_b32_e32 v0, s2
189; GFX7-NEXT:    v_mov_b32_e32 v1, s3
190; GFX7-NEXT:    s_waitcnt vmcnt(0)
191; GFX7-NEXT:    flat_store_dword v[0:1], v2
192; GFX7-NEXT:    s_endpgm
193;
194; GFX10-WGP-LABEL: flat_workgroup_acquire_load:
195; GFX10-WGP:       ; %bb.0: ; %entry
196; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
197; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
198; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
199; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
200; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
201; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
202; GFX10-WGP-NEXT:    buffer_gl0_inv
203; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
204; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
205; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
206; GFX10-WGP-NEXT:    s_endpgm
207;
208; GFX10-CU-LABEL: flat_workgroup_acquire_load:
209; GFX10-CU:       ; %bb.0: ; %entry
210; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
211; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
212; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
213; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
214; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
215; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
216; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
217; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
218; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
219; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
220; GFX10-CU-NEXT:    s_endpgm
221;
222; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_load:
223; SKIP-CACHE-INV:       ; %bb.0: ; %entry
224; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
225; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
226; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
227; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
228; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
229; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
230; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
231; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
232; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
233; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
234; SKIP-CACHE-INV-NEXT:    s_endpgm
235;
236; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_load:
237; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
238; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
239; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
240; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
241; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
242; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
243; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
244; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
245; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
246; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
247; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
248; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
249;
250; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_load:
251; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
252; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
253; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
254; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
255; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
256; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
257; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
258; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
259; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
260; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
261; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
262; GFX90A-TGSPLIT-NEXT:    s_endpgm
263    i32* %in, i32* %out) {
264entry:
265  %val = load atomic i32, i32* %in syncscope("workgroup") acquire, align 4
266  store i32 %val, i32* %out
267  ret void
268}
269
270define amdgpu_kernel void @flat_workgroup_seq_cst_load(
271; GFX7-LABEL: flat_workgroup_seq_cst_load:
272; GFX7:       ; %bb.0: ; %entry
273; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
274; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
275; GFX7-NEXT:    v_mov_b32_e32 v0, s0
276; GFX7-NEXT:    v_mov_b32_e32 v1, s1
277; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
278; GFX7-NEXT:    flat_load_dword v2, v[0:1]
279; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
280; GFX7-NEXT:    v_mov_b32_e32 v0, s2
281; GFX7-NEXT:    v_mov_b32_e32 v1, s3
282; GFX7-NEXT:    s_waitcnt vmcnt(0)
283; GFX7-NEXT:    flat_store_dword v[0:1], v2
284; GFX7-NEXT:    s_endpgm
285;
286; GFX10-WGP-LABEL: flat_workgroup_seq_cst_load:
287; GFX10-WGP:       ; %bb.0: ; %entry
288; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
289; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
290; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
291; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
292; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
293; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
294; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
295; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
296; GFX10-WGP-NEXT:    buffer_gl0_inv
297; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
298; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
299; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
300; GFX10-WGP-NEXT:    s_endpgm
301;
302; GFX10-CU-LABEL: flat_workgroup_seq_cst_load:
303; GFX10-CU:       ; %bb.0: ; %entry
304; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
305; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
306; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
307; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
308; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
309; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
310; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
311; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
312; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
313; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
314; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
315; GFX10-CU-NEXT:    s_endpgm
316;
317; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_load:
318; SKIP-CACHE-INV:       ; %bb.0: ; %entry
319; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
320; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
321; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
322; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
323; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
324; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
325; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
326; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
327; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
328; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
329; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
330; SKIP-CACHE-INV-NEXT:    s_endpgm
331;
332; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_load:
333; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
334; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
335; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
336; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
337; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
338; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
339; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
340; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
341; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
342; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
343; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
344; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
345; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
346;
347; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_load:
348; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
349; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
350; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
351; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
352; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
353; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
354; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
355; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
356; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
357; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
358; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
359; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
360; GFX90A-TGSPLIT-NEXT:    s_endpgm
361    i32* %in, i32* %out) {
362entry:
363  %val = load atomic i32, i32* %in syncscope("workgroup") seq_cst, align 4
364  store i32 %val, i32* %out
365  ret void
366}
367
368define amdgpu_kernel void @flat_workgroup_unordered_store(
369; GFX7-LABEL: flat_workgroup_unordered_store:
370; GFX7:       ; %bb.0: ; %entry
371; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
372; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
373; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
374; GFX7-NEXT:    v_mov_b32_e32 v0, s0
375; GFX7-NEXT:    v_mov_b32_e32 v1, s1
376; GFX7-NEXT:    v_mov_b32_e32 v2, s2
377; GFX7-NEXT:    flat_store_dword v[0:1], v2
378; GFX7-NEXT:    s_endpgm
379;
380; GFX10-WGP-LABEL: flat_workgroup_unordered_store:
381; GFX10-WGP:       ; %bb.0: ; %entry
382; GFX10-WGP-NEXT:    s_clause 0x1
383; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
384; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
385; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
386; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
387; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
388; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
389; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
390; GFX10-WGP-NEXT:    s_endpgm
391;
392; GFX10-CU-LABEL: flat_workgroup_unordered_store:
393; GFX10-CU:       ; %bb.0: ; %entry
394; GFX10-CU-NEXT:    s_clause 0x1
395; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
396; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
397; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
398; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
399; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
400; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
401; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
402; GFX10-CU-NEXT:    s_endpgm
403;
404; SKIP-CACHE-INV-LABEL: flat_workgroup_unordered_store:
405; SKIP-CACHE-INV:       ; %bb.0: ; %entry
406; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
407; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
408; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
409; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
410; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
411; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
412; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
413; SKIP-CACHE-INV-NEXT:    s_endpgm
414;
415; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_unordered_store:
416; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
417; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
418; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
419; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
420; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
421; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
422; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
423; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
424;
425; GFX90A-TGSPLIT-LABEL: flat_workgroup_unordered_store:
426; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
427; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
428; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
429; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
430; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
431; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
432; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
433; GFX90A-TGSPLIT-NEXT:    s_endpgm
434    i32 %in, i32* %out) {
435entry:
436  store atomic i32 %in, i32* %out syncscope("workgroup") unordered, align 4
437  ret void
438}
439
440define amdgpu_kernel void @flat_workgroup_monotonic_store(
441; GFX7-LABEL: flat_workgroup_monotonic_store:
442; GFX7:       ; %bb.0: ; %entry
443; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
444; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
445; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
446; GFX7-NEXT:    v_mov_b32_e32 v0, s0
447; GFX7-NEXT:    v_mov_b32_e32 v1, s1
448; GFX7-NEXT:    v_mov_b32_e32 v2, s2
449; GFX7-NEXT:    flat_store_dword v[0:1], v2
450; GFX7-NEXT:    s_endpgm
451;
452; GFX10-WGP-LABEL: flat_workgroup_monotonic_store:
453; GFX10-WGP:       ; %bb.0: ; %entry
454; GFX10-WGP-NEXT:    s_clause 0x1
455; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
456; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
457; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
458; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
459; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
460; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
461; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
462; GFX10-WGP-NEXT:    s_endpgm
463;
464; GFX10-CU-LABEL: flat_workgroup_monotonic_store:
465; GFX10-CU:       ; %bb.0: ; %entry
466; GFX10-CU-NEXT:    s_clause 0x1
467; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
468; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
469; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
470; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
471; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
472; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
473; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
474; GFX10-CU-NEXT:    s_endpgm
475;
476; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_store:
477; SKIP-CACHE-INV:       ; %bb.0: ; %entry
478; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
479; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
480; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
481; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
482; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
483; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
484; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
485; SKIP-CACHE-INV-NEXT:    s_endpgm
486;
487; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_store:
488; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
489; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
490; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
491; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
492; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
493; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
494; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
495; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
496;
497; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_store:
498; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
499; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
500; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
501; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
502; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
503; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
504; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
505; GFX90A-TGSPLIT-NEXT:    s_endpgm
506    i32 %in, i32* %out) {
507entry:
508  store atomic i32 %in, i32* %out syncscope("workgroup") monotonic, align 4
509  ret void
510}
511
512define amdgpu_kernel void @flat_workgroup_release_store(
513; GFX7-LABEL: flat_workgroup_release_store:
514; GFX7:       ; %bb.0: ; %entry
515; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
516; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
517; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
518; GFX7-NEXT:    v_mov_b32_e32 v0, s0
519; GFX7-NEXT:    v_mov_b32_e32 v1, s1
520; GFX7-NEXT:    v_mov_b32_e32 v2, s2
521; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
522; GFX7-NEXT:    flat_store_dword v[0:1], v2
523; GFX7-NEXT:    s_endpgm
524;
525; GFX10-WGP-LABEL: flat_workgroup_release_store:
526; GFX10-WGP:       ; %bb.0: ; %entry
527; GFX10-WGP-NEXT:    s_clause 0x1
528; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
529; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
530; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
531; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
532; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
533; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
534; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
535; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
536; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
537; GFX10-WGP-NEXT:    s_endpgm
538;
539; GFX10-CU-LABEL: flat_workgroup_release_store:
540; GFX10-CU:       ; %bb.0: ; %entry
541; GFX10-CU-NEXT:    s_clause 0x1
542; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
543; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
544; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
545; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
546; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
547; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
548; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
549; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
550; GFX10-CU-NEXT:    s_endpgm
551;
552; SKIP-CACHE-INV-LABEL: flat_workgroup_release_store:
553; SKIP-CACHE-INV:       ; %bb.0: ; %entry
554; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
555; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
556; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
557; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
558; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
559; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
560; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
561; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
562; SKIP-CACHE-INV-NEXT:    s_endpgm
563;
564; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_store:
565; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
566; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
567; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
568; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
569; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
570; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
571; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
572; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
573; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
574;
575; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_store:
576; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
577; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
578; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
579; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
580; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
581; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
582; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
583; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
584; GFX90A-TGSPLIT-NEXT:    s_endpgm
585    i32 %in, i32* %out) {
586entry:
587  store atomic i32 %in, i32* %out syncscope("workgroup") release, align 4
588  ret void
589}
590
591define amdgpu_kernel void @flat_workgroup_seq_cst_store(
592; GFX7-LABEL: flat_workgroup_seq_cst_store:
593; GFX7:       ; %bb.0: ; %entry
594; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
595; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
596; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
597; GFX7-NEXT:    v_mov_b32_e32 v0, s0
598; GFX7-NEXT:    v_mov_b32_e32 v1, s1
599; GFX7-NEXT:    v_mov_b32_e32 v2, s2
600; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
601; GFX7-NEXT:    flat_store_dword v[0:1], v2
602; GFX7-NEXT:    s_endpgm
603;
604; GFX10-WGP-LABEL: flat_workgroup_seq_cst_store:
605; GFX10-WGP:       ; %bb.0: ; %entry
606; GFX10-WGP-NEXT:    s_clause 0x1
607; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
608; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
609; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
610; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
611; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
612; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
613; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
614; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
615; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
616; GFX10-WGP-NEXT:    s_endpgm
617;
618; GFX10-CU-LABEL: flat_workgroup_seq_cst_store:
619; GFX10-CU:       ; %bb.0: ; %entry
620; GFX10-CU-NEXT:    s_clause 0x1
621; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
622; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
623; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
624; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
625; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
626; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
627; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
628; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
629; GFX10-CU-NEXT:    s_endpgm
630;
631; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_store:
632; SKIP-CACHE-INV:       ; %bb.0: ; %entry
633; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
634; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
635; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
636; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
637; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
638; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
639; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
640; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
641; SKIP-CACHE-INV-NEXT:    s_endpgm
642;
643; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_store:
644; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
645; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
646; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
647; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
648; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
649; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
650; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
651; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
652; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
653;
654; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_store:
655; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
656; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
657; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
658; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
659; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
660; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
661; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
662; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
663; GFX90A-TGSPLIT-NEXT:    s_endpgm
664    i32 %in, i32* %out) {
665entry:
666  store atomic i32 %in, i32* %out syncscope("workgroup") seq_cst, align 4
667  ret void
668}
669
670define amdgpu_kernel void @flat_workgroup_monotonic_atomicrmw(
671; GFX7-LABEL: flat_workgroup_monotonic_atomicrmw:
672; GFX7:       ; %bb.0: ; %entry
673; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
674; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
675; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
676; GFX7-NEXT:    v_mov_b32_e32 v0, s0
677; GFX7-NEXT:    v_mov_b32_e32 v1, s1
678; GFX7-NEXT:    v_mov_b32_e32 v2, s2
679; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
680; GFX7-NEXT:    s_endpgm
681;
682; GFX10-WGP-LABEL: flat_workgroup_monotonic_atomicrmw:
683; GFX10-WGP:       ; %bb.0: ; %entry
684; GFX10-WGP-NEXT:    s_clause 0x1
685; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
686; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
687; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
688; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
689; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
690; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
691; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
692; GFX10-WGP-NEXT:    s_endpgm
693;
694; GFX10-CU-LABEL: flat_workgroup_monotonic_atomicrmw:
695; GFX10-CU:       ; %bb.0: ; %entry
696; GFX10-CU-NEXT:    s_clause 0x1
697; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
698; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
699; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
700; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
701; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
702; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
703; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
704; GFX10-CU-NEXT:    s_endpgm
705;
706; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_atomicrmw:
707; SKIP-CACHE-INV:       ; %bb.0: ; %entry
708; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
709; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
710; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
711; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
712; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
713; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
714; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
715; SKIP-CACHE-INV-NEXT:    s_endpgm
716;
717; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw:
718; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
719; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
720; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
721; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
722; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
723; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
724; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
725; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
726;
727; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw:
728; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
729; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
730; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
731; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
732; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
733; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
734; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
735; GFX90A-TGSPLIT-NEXT:    s_endpgm
736    i32* %out, i32 %in) {
737entry:
738  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") monotonic
739  ret void
740}
741
742define amdgpu_kernel void @flat_workgroup_acquire_atomicrmw(
743; GFX7-LABEL: flat_workgroup_acquire_atomicrmw:
744; GFX7:       ; %bb.0: ; %entry
745; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
746; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
747; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
748; GFX7-NEXT:    v_mov_b32_e32 v0, s0
749; GFX7-NEXT:    v_mov_b32_e32 v1, s1
750; GFX7-NEXT:    v_mov_b32_e32 v2, s2
751; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
752; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
753; GFX7-NEXT:    s_endpgm
754;
755; GFX10-WGP-LABEL: flat_workgroup_acquire_atomicrmw:
756; GFX10-WGP:       ; %bb.0: ; %entry
757; GFX10-WGP-NEXT:    s_clause 0x1
758; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
759; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
760; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
761; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
762; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
763; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
764; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
765; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
766; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
767; GFX10-WGP-NEXT:    buffer_gl0_inv
768; GFX10-WGP-NEXT:    s_endpgm
769;
770; GFX10-CU-LABEL: flat_workgroup_acquire_atomicrmw:
771; GFX10-CU:       ; %bb.0: ; %entry
772; GFX10-CU-NEXT:    s_clause 0x1
773; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
774; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
775; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
776; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
777; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
778; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
779; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
780; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
781; GFX10-CU-NEXT:    s_endpgm
782;
783; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_atomicrmw:
784; SKIP-CACHE-INV:       ; %bb.0: ; %entry
785; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
786; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
787; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
788; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
789; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
790; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
791; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
792; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
793; SKIP-CACHE-INV-NEXT:    s_endpgm
794;
795; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw:
796; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
797; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
798; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
799; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
800; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
801; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
802; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
803; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
804; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
805;
806; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw:
807; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
808; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
809; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
810; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
811; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
812; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
813; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
814; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
815; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
816; GFX90A-TGSPLIT-NEXT:    s_endpgm
817    i32* %out, i32 %in) {
818entry:
819  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acquire
820  ret void
821}
822
823define amdgpu_kernel void @flat_workgroup_release_atomicrmw(
824; GFX7-LABEL: flat_workgroup_release_atomicrmw:
825; GFX7:       ; %bb.0: ; %entry
826; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
827; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
828; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
829; GFX7-NEXT:    v_mov_b32_e32 v0, s0
830; GFX7-NEXT:    v_mov_b32_e32 v1, s1
831; GFX7-NEXT:    v_mov_b32_e32 v2, s2
832; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
833; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
834; GFX7-NEXT:    s_endpgm
835;
836; GFX10-WGP-LABEL: flat_workgroup_release_atomicrmw:
837; GFX10-WGP:       ; %bb.0: ; %entry
838; GFX10-WGP-NEXT:    s_clause 0x1
839; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
840; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
841; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
842; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
843; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
844; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
845; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
846; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
847; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
848; GFX10-WGP-NEXT:    s_endpgm
849;
850; GFX10-CU-LABEL: flat_workgroup_release_atomicrmw:
851; GFX10-CU:       ; %bb.0: ; %entry
852; GFX10-CU-NEXT:    s_clause 0x1
853; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
854; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
855; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
856; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
857; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
858; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
859; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
860; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
861; GFX10-CU-NEXT:    s_endpgm
862;
863; SKIP-CACHE-INV-LABEL: flat_workgroup_release_atomicrmw:
864; SKIP-CACHE-INV:       ; %bb.0: ; %entry
865; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
866; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
867; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
868; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
869; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
870; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
871; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
872; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
873; SKIP-CACHE-INV-NEXT:    s_endpgm
874;
875; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_atomicrmw:
876; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
877; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
878; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
879; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
880; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
881; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
882; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
883; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
884; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
885;
886; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_atomicrmw:
887; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
888; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
889; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
890; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
891; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
892; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
893; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
894; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
895; GFX90A-TGSPLIT-NEXT:    s_endpgm
896    i32* %out, i32 %in) {
897entry:
898  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") release
899  ret void
900}
901
902define amdgpu_kernel void @flat_workgroup_acq_rel_atomicrmw(
903; GFX7-LABEL: flat_workgroup_acq_rel_atomicrmw:
904; GFX7:       ; %bb.0: ; %entry
905; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
906; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
907; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
908; GFX7-NEXT:    v_mov_b32_e32 v0, s0
909; GFX7-NEXT:    v_mov_b32_e32 v1, s1
910; GFX7-NEXT:    v_mov_b32_e32 v2, s2
911; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
912; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
913; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
914; GFX7-NEXT:    s_endpgm
915;
916; GFX10-WGP-LABEL: flat_workgroup_acq_rel_atomicrmw:
917; GFX10-WGP:       ; %bb.0: ; %entry
918; GFX10-WGP-NEXT:    s_clause 0x1
919; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
920; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
921; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
922; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
923; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
924; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
925; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
926; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
927; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
928; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
929; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
930; GFX10-WGP-NEXT:    buffer_gl0_inv
931; GFX10-WGP-NEXT:    s_endpgm
932;
933; GFX10-CU-LABEL: flat_workgroup_acq_rel_atomicrmw:
934; GFX10-CU:       ; %bb.0: ; %entry
935; GFX10-CU-NEXT:    s_clause 0x1
936; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
937; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
938; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
939; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
940; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
941; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
942; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
943; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
944; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
945; GFX10-CU-NEXT:    s_endpgm
946;
947; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_atomicrmw:
948; SKIP-CACHE-INV:       ; %bb.0: ; %entry
949; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
950; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
951; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
952; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
953; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
954; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
955; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
956; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
957; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
958; SKIP-CACHE-INV-NEXT:    s_endpgm
959;
960; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw:
961; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
962; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
963; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
964; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
965; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
966; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
967; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
968; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
969; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
970; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
971;
972; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw:
973; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
974; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
975; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
976; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
977; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
978; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
979; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
980; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
981; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
982; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
983; GFX90A-TGSPLIT-NEXT:    s_endpgm
984    i32* %out, i32 %in) {
985entry:
986  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acq_rel
987  ret void
988}
989
990define amdgpu_kernel void @flat_workgroup_seq_cst_atomicrmw(
991; GFX7-LABEL: flat_workgroup_seq_cst_atomicrmw:
992; GFX7:       ; %bb.0: ; %entry
993; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
994; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
995; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
996; GFX7-NEXT:    v_mov_b32_e32 v0, s0
997; GFX7-NEXT:    v_mov_b32_e32 v1, s1
998; GFX7-NEXT:    v_mov_b32_e32 v2, s2
999; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1000; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1001; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1002; GFX7-NEXT:    s_endpgm
1003;
1004; GFX10-WGP-LABEL: flat_workgroup_seq_cst_atomicrmw:
1005; GFX10-WGP:       ; %bb.0: ; %entry
1006; GFX10-WGP-NEXT:    s_clause 0x1
1007; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1008; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1009; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1010; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1011; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1012; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1013; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1014; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1015; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1016; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1017; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1018; GFX10-WGP-NEXT:    buffer_gl0_inv
1019; GFX10-WGP-NEXT:    s_endpgm
1020;
1021; GFX10-CU-LABEL: flat_workgroup_seq_cst_atomicrmw:
1022; GFX10-CU:       ; %bb.0: ; %entry
1023; GFX10-CU-NEXT:    s_clause 0x1
1024; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1025; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1026; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1027; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1028; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1029; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1030; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1031; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1032; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1033; GFX10-CU-NEXT:    s_endpgm
1034;
1035; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_atomicrmw:
1036; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1037; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1038; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1039; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1040; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1041; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1042; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1043; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1044; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1045; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1046; SKIP-CACHE-INV-NEXT:    s_endpgm
1047;
1048; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw:
1049; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1050; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1051; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1052; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1053; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1054; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1055; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1056; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1057; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1058; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1059;
1060; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw:
1061; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1062; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1063; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1064; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1065; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1066; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1067; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1068; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1069; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1070; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1071; GFX90A-TGSPLIT-NEXT:    s_endpgm
1072    i32* %out, i32 %in) {
1073entry:
1074  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") seq_cst
1075  ret void
1076}
1077
1078define amdgpu_kernel void @flat_workgroup_acquire_ret_atomicrmw(
1079; GFX7-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1080; GFX7:       ; %bb.0: ; %entry
1081; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1082; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1083; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1084; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1085; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1086; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1087; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1088; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1089; GFX7-NEXT:    flat_store_dword v[0:1], v2
1090; GFX7-NEXT:    s_endpgm
1091;
1092; GFX10-WGP-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1093; GFX10-WGP:       ; %bb.0: ; %entry
1094; GFX10-WGP-NEXT:    s_clause 0x1
1095; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1096; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1097; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1098; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1099; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1100; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1101; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1102; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1103; GFX10-WGP-NEXT:    buffer_gl0_inv
1104; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1105; GFX10-WGP-NEXT:    s_endpgm
1106;
1107; GFX10-CU-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1108; GFX10-CU:       ; %bb.0: ; %entry
1109; GFX10-CU-NEXT:    s_clause 0x1
1110; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1111; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1112; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1113; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1114; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1115; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1116; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1117; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1118; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1119; GFX10-CU-NEXT:    s_endpgm
1120;
1121; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1122; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1123; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1124; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1125; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1126; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1127; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1128; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1129; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1130; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1131; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1132; SKIP-CACHE-INV-NEXT:    s_endpgm
1133;
1134; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1135; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1136; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1137; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1138; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1139; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1140; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1141; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1142; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1143; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1144; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1145;
1146; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1147; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1148; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1149; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1150; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1151; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1152; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1153; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1154; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1155; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1156; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1157; GFX90A-TGSPLIT-NEXT:    s_endpgm
1158    i32* %out, i32 %in) {
1159entry:
1160  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acquire
1161  store i32 %val, i32* %out, align 4
1162  ret void
1163}
1164
1165define amdgpu_kernel void @flat_workgroup_acq_rel_ret_atomicrmw(
1166; GFX7-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1167; GFX7:       ; %bb.0: ; %entry
1168; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1169; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1170; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1171; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1172; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1173; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1174; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1175; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1176; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1177; GFX7-NEXT:    flat_store_dword v[0:1], v2
1178; GFX7-NEXT:    s_endpgm
1179;
1180; GFX10-WGP-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1181; GFX10-WGP:       ; %bb.0: ; %entry
1182; GFX10-WGP-NEXT:    s_clause 0x1
1183; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1184; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1185; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1186; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1187; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1188; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1189; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1190; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1191; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1192; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1193; GFX10-WGP-NEXT:    buffer_gl0_inv
1194; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1195; GFX10-WGP-NEXT:    s_endpgm
1196;
1197; GFX10-CU-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1198; GFX10-CU:       ; %bb.0: ; %entry
1199; GFX10-CU-NEXT:    s_clause 0x1
1200; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1201; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1202; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1203; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1204; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1205; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1206; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1207; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1208; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1209; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1210; GFX10-CU-NEXT:    s_endpgm
1211;
1212; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1213; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1214; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1215; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1216; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1217; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1218; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1219; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1220; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1221; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1222; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1223; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1224; SKIP-CACHE-INV-NEXT:    s_endpgm
1225;
1226; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1227; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1228; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1229; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1230; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1231; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1232; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1233; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1234; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1235; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1236; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1237; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1238;
1239; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1240; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1241; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1242; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1243; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1244; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1245; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1246; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1247; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1248; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1249; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1250; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1251; GFX90A-TGSPLIT-NEXT:    s_endpgm
1252    i32* %out, i32 %in) {
1253entry:
1254  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acq_rel
1255  store i32 %val, i32* %out, align 4
1256  ret void
1257}
1258
1259define amdgpu_kernel void @flat_workgroup_seq_cst_ret_atomicrmw(
1260; GFX7-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
1261; GFX7:       ; %bb.0: ; %entry
1262; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1263; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1264; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1265; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1266; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1267; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1268; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1269; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1270; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1271; GFX7-NEXT:    flat_store_dword v[0:1], v2
1272; GFX7-NEXT:    s_endpgm
1273;
1274; GFX10-WGP-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
1275; GFX10-WGP:       ; %bb.0: ; %entry
1276; GFX10-WGP-NEXT:    s_clause 0x1
1277; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1278; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1279; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1280; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1281; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1282; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1283; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1284; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1285; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1286; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1287; GFX10-WGP-NEXT:    buffer_gl0_inv
1288; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1289; GFX10-WGP-NEXT:    s_endpgm
1290;
1291; GFX10-CU-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
1292; GFX10-CU:       ; %bb.0: ; %entry
1293; GFX10-CU-NEXT:    s_clause 0x1
1294; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1295; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1296; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1297; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1298; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1299; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1300; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1301; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1302; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1303; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1304; GFX10-CU-NEXT:    s_endpgm
1305;
1306; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
1307; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1308; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1309; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1310; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1311; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1312; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1313; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1314; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1315; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1316; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1317; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1318; SKIP-CACHE-INV-NEXT:    s_endpgm
1319;
1320; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
1321; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1322; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1323; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1324; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1325; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1326; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1327; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1328; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1329; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1330; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1331; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1332;
1333; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
1334; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1335; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1336; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1337; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1338; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1339; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1340; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1341; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1342; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1343; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1344; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1345; GFX90A-TGSPLIT-NEXT:    s_endpgm
1346    i32* %out, i32 %in) {
1347entry:
1348  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") seq_cst
1349  store i32 %val, i32* %out, align 4
1350  ret void
1351}
1352
1353define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_cmpxchg(
1354; GFX7-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
1355; GFX7:       ; %bb.0: ; %entry
1356; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1357; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1358; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1359; GFX7-NEXT:    s_add_u32 s0, s0, 16
1360; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1361; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1362; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1363; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1364; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1365; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1366; GFX7-NEXT:    s_endpgm
1367;
1368; GFX10-WGP-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
1369; GFX10-WGP:       ; %bb.0: ; %entry
1370; GFX10-WGP-NEXT:    s_clause 0x1
1371; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1372; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1373; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1374; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1375; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1376; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1377; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1378; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1379; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1380; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1381; GFX10-WGP-NEXT:    s_endpgm
1382;
1383; GFX10-CU-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
1384; GFX10-CU:       ; %bb.0: ; %entry
1385; GFX10-CU-NEXT:    s_clause 0x1
1386; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1387; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1388; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1389; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1390; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1391; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1392; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1393; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1394; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1395; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1396; GFX10-CU-NEXT:    s_endpgm
1397;
1398; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
1399; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1400; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1401; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
1402; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1403; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1404; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1405; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1406; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1407; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1408; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1409; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1410; SKIP-CACHE-INV-NEXT:    s_endpgm
1411;
1412; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
1413; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1414; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1415; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1416; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1417; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1418; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1419; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1420; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1421;
1422; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
1423; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1424; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1425; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1426; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1427; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1428; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1429; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1430; GFX90A-TGSPLIT-NEXT:    s_endpgm
1431    i32* %out, i32 %in, i32 %old) {
1432entry:
1433  %gep = getelementptr i32, i32* %out, i32 4
1434  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic
1435  ret void
1436}
1437
1438define amdgpu_kernel void @flat_workgroup_acquire_monotonic_cmpxchg(
1439; GFX7-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
1440; GFX7:       ; %bb.0: ; %entry
1441; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1442; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1443; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1444; GFX7-NEXT:    s_add_u32 s0, s0, 16
1445; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1446; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1447; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1448; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1449; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1450; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1451; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1452; GFX7-NEXT:    s_endpgm
1453;
1454; GFX10-WGP-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
1455; GFX10-WGP:       ; %bb.0: ; %entry
1456; GFX10-WGP-NEXT:    s_clause 0x1
1457; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1458; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1459; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1460; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1461; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1462; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1463; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1464; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1465; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1466; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1467; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1468; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1469; GFX10-WGP-NEXT:    buffer_gl0_inv
1470; GFX10-WGP-NEXT:    s_endpgm
1471;
1472; GFX10-CU-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
1473; GFX10-CU:       ; %bb.0: ; %entry
1474; GFX10-CU-NEXT:    s_clause 0x1
1475; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1476; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1477; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1478; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1479; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1480; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1481; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1482; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1483; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1484; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1485; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1486; GFX10-CU-NEXT:    s_endpgm
1487;
1488; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
1489; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1490; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1491; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
1492; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1493; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1494; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1495; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1496; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1497; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1498; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1499; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1500; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1501; SKIP-CACHE-INV-NEXT:    s_endpgm
1502;
1503; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
1504; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1505; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1506; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1507; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1508; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1509; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1510; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1511; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1512; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1513;
1514; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
1515; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1516; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1517; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1518; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1519; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1520; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1521; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1522; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1523; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1524; GFX90A-TGSPLIT-NEXT:    s_endpgm
1525    i32* %out, i32 %in, i32 %old) {
1526entry:
1527  %gep = getelementptr i32, i32* %out, i32 4
1528  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
1529  ret void
1530}
1531
1532define amdgpu_kernel void @flat_workgroup_release_monotonic_cmpxchg(
1533; GFX7-LABEL: flat_workgroup_release_monotonic_cmpxchg:
1534; GFX7:       ; %bb.0: ; %entry
1535; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1536; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1537; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1538; GFX7-NEXT:    s_add_u32 s0, s0, 16
1539; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1540; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1541; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1542; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1543; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1544; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1545; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1546; GFX7-NEXT:    s_endpgm
1547;
1548; GFX10-WGP-LABEL: flat_workgroup_release_monotonic_cmpxchg:
1549; GFX10-WGP:       ; %bb.0: ; %entry
1550; GFX10-WGP-NEXT:    s_clause 0x1
1551; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1552; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1553; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1554; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1555; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1556; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1557; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1558; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1559; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1560; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1561; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1562; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1563; GFX10-WGP-NEXT:    s_endpgm
1564;
1565; GFX10-CU-LABEL: flat_workgroup_release_monotonic_cmpxchg:
1566; GFX10-CU:       ; %bb.0: ; %entry
1567; GFX10-CU-NEXT:    s_clause 0x1
1568; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1569; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1570; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1571; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1572; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1573; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1574; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1575; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1576; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1577; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1578; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1579; GFX10-CU-NEXT:    s_endpgm
1580;
1581; SKIP-CACHE-INV-LABEL: flat_workgroup_release_monotonic_cmpxchg:
1582; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1583; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1584; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
1585; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1586; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1587; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1588; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1589; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1590; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1591; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1592; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1593; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1594; SKIP-CACHE-INV-NEXT:    s_endpgm
1595;
1596; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg:
1597; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1598; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1599; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1600; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1601; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1602; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1603; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1604; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1605; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1606;
1607; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg:
1608; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1609; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1610; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1611; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1612; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1613; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1614; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1615; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1616; GFX90A-TGSPLIT-NEXT:    s_endpgm
1617    i32* %out, i32 %in, i32 %old) {
1618entry:
1619  %gep = getelementptr i32, i32* %out, i32 4
1620  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic
1621  ret void
1622}
1623
1624define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_cmpxchg(
1625; GFX7-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
1626; GFX7:       ; %bb.0: ; %entry
1627; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1628; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1629; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1630; GFX7-NEXT:    s_add_u32 s0, s0, 16
1631; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1632; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1633; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1634; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1635; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1636; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1637; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1638; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1639; GFX7-NEXT:    s_endpgm
1640;
1641; GFX10-WGP-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
1642; GFX10-WGP:       ; %bb.0: ; %entry
1643; GFX10-WGP-NEXT:    s_clause 0x1
1644; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1645; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1646; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1647; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1648; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1649; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1650; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1651; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1652; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1653; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1654; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1655; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1656; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1657; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1658; GFX10-WGP-NEXT:    buffer_gl0_inv
1659; GFX10-WGP-NEXT:    s_endpgm
1660;
1661; GFX10-CU-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
1662; GFX10-CU:       ; %bb.0: ; %entry
1663; GFX10-CU-NEXT:    s_clause 0x1
1664; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1665; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1666; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1667; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1668; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1669; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1670; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1671; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1672; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1673; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1674; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1675; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1676; GFX10-CU-NEXT:    s_endpgm
1677;
1678; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
1679; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1680; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1681; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
1682; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1683; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1684; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1685; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1686; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1687; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1688; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1689; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1690; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1691; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1692; SKIP-CACHE-INV-NEXT:    s_endpgm
1693;
1694; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
1695; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1696; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1697; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1698; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1699; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1700; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1701; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1702; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1703; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1704; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1705;
1706; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
1707; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1708; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1709; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1710; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1711; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1712; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1713; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1714; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1715; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1716; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1717; GFX90A-TGSPLIT-NEXT:    s_endpgm
1718    i32* %out, i32 %in, i32 %old) {
1719entry:
1720  %gep = getelementptr i32, i32* %out, i32 4
1721  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
1722  ret void
1723}
1724
1725define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_cmpxchg(
1726; GFX7-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
1727; GFX7:       ; %bb.0: ; %entry
1728; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1729; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1730; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1731; GFX7-NEXT:    s_add_u32 s0, s0, 16
1732; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1733; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1734; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1735; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1736; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1737; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1738; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1739; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1740; GFX7-NEXT:    s_endpgm
1741;
1742; GFX10-WGP-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
1743; GFX10-WGP:       ; %bb.0: ; %entry
1744; GFX10-WGP-NEXT:    s_clause 0x1
1745; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1746; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1747; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1748; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1749; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1750; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1751; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1752; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1753; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1754; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1755; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1756; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1757; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1758; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1759; GFX10-WGP-NEXT:    buffer_gl0_inv
1760; GFX10-WGP-NEXT:    s_endpgm
1761;
1762; GFX10-CU-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
1763; GFX10-CU:       ; %bb.0: ; %entry
1764; GFX10-CU-NEXT:    s_clause 0x1
1765; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1766; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1767; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1768; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1769; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1770; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1771; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1772; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1773; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1774; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1775; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1776; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1777; GFX10-CU-NEXT:    s_endpgm
1778;
1779; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
1780; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1781; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1782; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
1783; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1784; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1785; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1786; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1787; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1788; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1789; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1790; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1791; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1792; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1793; SKIP-CACHE-INV-NEXT:    s_endpgm
1794;
1795; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
1796; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1797; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1798; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1799; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1800; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1801; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1802; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1803; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1804; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1805; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1806;
1807; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
1808; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1809; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1810; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1811; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1812; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1813; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1814; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1815; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1816; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1817; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1818; GFX90A-TGSPLIT-NEXT:    s_endpgm
1819    i32* %out, i32 %in, i32 %old) {
1820entry:
1821  %gep = getelementptr i32, i32* %out, i32 4
1822  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
1823  ret void
1824}
1825
1826define amdgpu_kernel void @flat_workgroup_monotonic_acquire_cmpxchg(
1827; GFX7-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
1828; GFX7:       ; %bb.0: ; %entry
1829; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1830; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1831; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1832; GFX7-NEXT:    s_add_u32 s0, s0, 16
1833; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1834; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1835; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1836; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1837; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1838; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1839; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1840; GFX7-NEXT:    s_endpgm
1841;
1842; GFX10-WGP-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
1843; GFX10-WGP:       ; %bb.0: ; %entry
1844; GFX10-WGP-NEXT:    s_clause 0x1
1845; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1846; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1847; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1848; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1849; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1850; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1851; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1852; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1853; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1854; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1855; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1856; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1857; GFX10-WGP-NEXT:    buffer_gl0_inv
1858; GFX10-WGP-NEXT:    s_endpgm
1859;
1860; GFX10-CU-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
1861; GFX10-CU:       ; %bb.0: ; %entry
1862; GFX10-CU-NEXT:    s_clause 0x1
1863; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1864; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1865; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1866; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1867; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1868; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1869; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1870; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1871; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1872; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1873; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1874; GFX10-CU-NEXT:    s_endpgm
1875;
1876; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
1877; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1878; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1879; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
1880; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1881; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1882; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1883; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1884; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1885; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1886; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1887; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1888; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1889; SKIP-CACHE-INV-NEXT:    s_endpgm
1890;
1891; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
1892; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1893; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1894; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1895; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1896; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1897; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1898; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1899; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1900; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1901;
1902; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
1903; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1904; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1905; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1906; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1907; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1908; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1909; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1910; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1911; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1912; GFX90A-TGSPLIT-NEXT:    s_endpgm
1913    i32* %out, i32 %in, i32 %old) {
1914entry:
1915  %gep = getelementptr i32, i32* %out, i32 4
1916  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic acquire
1917  ret void
1918}
1919
1920define amdgpu_kernel void @flat_workgroup_acquire_acquire_cmpxchg(
1921; GFX7-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
1922; GFX7:       ; %bb.0: ; %entry
1923; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1924; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1925; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1926; GFX7-NEXT:    s_add_u32 s0, s0, 16
1927; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1928; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1929; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1930; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1931; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1932; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1933; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1934; GFX7-NEXT:    s_endpgm
1935;
1936; GFX10-WGP-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
1937; GFX10-WGP:       ; %bb.0: ; %entry
1938; GFX10-WGP-NEXT:    s_clause 0x1
1939; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1940; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1941; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1942; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1943; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1944; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1945; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1946; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1947; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1948; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1949; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1950; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1951; GFX10-WGP-NEXT:    buffer_gl0_inv
1952; GFX10-WGP-NEXT:    s_endpgm
1953;
1954; GFX10-CU-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
1955; GFX10-CU:       ; %bb.0: ; %entry
1956; GFX10-CU-NEXT:    s_clause 0x1
1957; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1958; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1959; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1960; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1961; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1962; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1963; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1964; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1965; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1966; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1967; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1968; GFX10-CU-NEXT:    s_endpgm
1969;
1970; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
1971; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1972; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1973; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
1974; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1975; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1976; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1977; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1978; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1979; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1980; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1981; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1982; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1983; SKIP-CACHE-INV-NEXT:    s_endpgm
1984;
1985; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
1986; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1987; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1988; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1989; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1990; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1991; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1992; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1993; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1994; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1995;
1996; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
1997; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1998; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1999; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2000; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2001; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2002; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2003; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2004; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2005; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2006; GFX90A-TGSPLIT-NEXT:    s_endpgm
2007    i32* %out, i32 %in, i32 %old) {
2008entry:
2009  %gep = getelementptr i32, i32* %out, i32 4
2010  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
2011  ret void
2012}
2013
2014define amdgpu_kernel void @flat_workgroup_release_acquire_cmpxchg(
2015; GFX7-LABEL: flat_workgroup_release_acquire_cmpxchg:
2016; GFX7:       ; %bb.0: ; %entry
2017; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2018; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2019; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2020; GFX7-NEXT:    s_add_u32 s0, s0, 16
2021; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2022; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2023; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2024; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2025; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2026; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2027; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2028; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2029; GFX7-NEXT:    s_endpgm
2030;
2031; GFX10-WGP-LABEL: flat_workgroup_release_acquire_cmpxchg:
2032; GFX10-WGP:       ; %bb.0: ; %entry
2033; GFX10-WGP-NEXT:    s_clause 0x1
2034; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2035; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2036; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2037; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2038; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2039; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2040; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2041; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2042; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2043; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2044; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2045; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2046; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2047; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2048; GFX10-WGP-NEXT:    buffer_gl0_inv
2049; GFX10-WGP-NEXT:    s_endpgm
2050;
2051; GFX10-CU-LABEL: flat_workgroup_release_acquire_cmpxchg:
2052; GFX10-CU:       ; %bb.0: ; %entry
2053; GFX10-CU-NEXT:    s_clause 0x1
2054; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2055; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2056; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2057; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2058; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2059; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2060; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2061; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2062; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2063; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2064; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2065; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2066; GFX10-CU-NEXT:    s_endpgm
2067;
2068; SKIP-CACHE-INV-LABEL: flat_workgroup_release_acquire_cmpxchg:
2069; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2070; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2071; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2072; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2073; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2074; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2075; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2076; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2077; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2078; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2079; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2080; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2081; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2082; SKIP-CACHE-INV-NEXT:    s_endpgm
2083;
2084; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg:
2085; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2086; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2087; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2088; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2089; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2090; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2091; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2092; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2093; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2094; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2095;
2096; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg:
2097; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2098; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2099; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2100; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2101; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2102; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2103; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2104; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2105; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2106; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2107; GFX90A-TGSPLIT-NEXT:    s_endpgm
2108    i32* %out, i32 %in, i32 %old) {
2109entry:
2110  %gep = getelementptr i32, i32* %out, i32 4
2111  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
2112  ret void
2113}
2114
2115define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_cmpxchg(
2116; GFX7-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
2117; GFX7:       ; %bb.0: ; %entry
2118; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2119; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2120; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2121; GFX7-NEXT:    s_add_u32 s0, s0, 16
2122; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2123; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2124; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2125; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2126; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2127; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2128; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2129; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2130; GFX7-NEXT:    s_endpgm
2131;
2132; GFX10-WGP-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
2133; GFX10-WGP:       ; %bb.0: ; %entry
2134; GFX10-WGP-NEXT:    s_clause 0x1
2135; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2136; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2137; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2138; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2139; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2140; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2141; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2142; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2143; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2144; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2145; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2146; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2147; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2148; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2149; GFX10-WGP-NEXT:    buffer_gl0_inv
2150; GFX10-WGP-NEXT:    s_endpgm
2151;
2152; GFX10-CU-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
2153; GFX10-CU:       ; %bb.0: ; %entry
2154; GFX10-CU-NEXT:    s_clause 0x1
2155; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2156; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2157; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2158; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2159; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2160; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2161; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2162; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2163; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2164; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2165; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2166; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2167; GFX10-CU-NEXT:    s_endpgm
2168;
2169; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
2170; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2171; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2172; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2173; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2174; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2175; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2176; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2177; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2178; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2179; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2180; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2181; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2182; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2183; SKIP-CACHE-INV-NEXT:    s_endpgm
2184;
2185; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
2186; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2187; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2188; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2189; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2190; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2191; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2192; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2193; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2194; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2195; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2196;
2197; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
2198; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2199; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2200; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2201; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2202; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2203; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2204; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2205; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2206; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2207; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2208; GFX90A-TGSPLIT-NEXT:    s_endpgm
2209    i32* %out, i32 %in, i32 %old) {
2210entry:
2211  %gep = getelementptr i32, i32* %out, i32 4
2212  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
2213  ret void
2214}
2215
2216define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_cmpxchg(
2217; GFX7-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
2218; GFX7:       ; %bb.0: ; %entry
2219; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2220; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2221; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2222; GFX7-NEXT:    s_add_u32 s0, s0, 16
2223; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2224; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2225; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2226; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2227; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2228; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2229; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2230; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2231; GFX7-NEXT:    s_endpgm
2232;
2233; GFX10-WGP-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
2234; GFX10-WGP:       ; %bb.0: ; %entry
2235; GFX10-WGP-NEXT:    s_clause 0x1
2236; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2237; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2238; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2239; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2240; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2241; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2242; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2243; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2244; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2245; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2246; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2247; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2248; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2249; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2250; GFX10-WGP-NEXT:    buffer_gl0_inv
2251; GFX10-WGP-NEXT:    s_endpgm
2252;
2253; GFX10-CU-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
2254; GFX10-CU:       ; %bb.0: ; %entry
2255; GFX10-CU-NEXT:    s_clause 0x1
2256; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2257; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2258; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2259; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2260; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2261; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2262; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2263; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2264; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2265; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2266; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2267; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2268; GFX10-CU-NEXT:    s_endpgm
2269;
2270; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
2271; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2272; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2273; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2274; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2275; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2276; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2277; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2278; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2279; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2280; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2281; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2282; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2283; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2284; SKIP-CACHE-INV-NEXT:    s_endpgm
2285;
2286; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
2287; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2288; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2289; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2290; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2291; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2292; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2293; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2294; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2295; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2296; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2297;
2298; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
2299; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2300; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2301; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2302; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2303; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2304; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2305; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2306; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2307; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2308; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2309; GFX90A-TGSPLIT-NEXT:    s_endpgm
2310    i32* %out, i32 %in, i32 %old) {
2311entry:
2312  %gep = getelementptr i32, i32* %out, i32 4
2313  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
2314  ret void
2315}
2316
2317define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_cmpxchg(
2318; GFX7-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
2319; GFX7:       ; %bb.0: ; %entry
2320; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2321; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2322; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2323; GFX7-NEXT:    s_add_u32 s0, s0, 16
2324; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2325; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2326; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2327; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2328; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2329; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2330; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2331; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2332; GFX7-NEXT:    s_endpgm
2333;
2334; GFX10-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
2335; GFX10-WGP:       ; %bb.0: ; %entry
2336; GFX10-WGP-NEXT:    s_clause 0x1
2337; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2338; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2339; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2340; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2341; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2342; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2343; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2344; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2345; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2346; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2347; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2348; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2349; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2350; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2351; GFX10-WGP-NEXT:    buffer_gl0_inv
2352; GFX10-WGP-NEXT:    s_endpgm
2353;
2354; GFX10-CU-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
2355; GFX10-CU:       ; %bb.0: ; %entry
2356; GFX10-CU-NEXT:    s_clause 0x1
2357; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2358; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2359; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2360; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2361; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2362; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2363; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2364; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2365; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2366; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2367; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2368; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2369; GFX10-CU-NEXT:    s_endpgm
2370;
2371; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
2372; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2373; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2374; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2375; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2376; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2377; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2378; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2379; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2380; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2381; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2382; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2383; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2384; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2385; SKIP-CACHE-INV-NEXT:    s_endpgm
2386;
2387; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
2388; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2389; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2390; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2391; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2392; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2393; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2394; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2395; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2396; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2397; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2398;
2399; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
2400; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2401; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2402; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2403; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2404; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2405; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2406; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2407; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2408; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2409; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2410; GFX90A-TGSPLIT-NEXT:    s_endpgm
2411    i32* %out, i32 %in, i32 %old) {
2412entry:
2413  %gep = getelementptr i32, i32* %out, i32 4
2414  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
2415  ret void
2416}
2417
2418define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_ret_cmpxchg(
2419; GFX7-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
2420; GFX7:       ; %bb.0: ; %entry
2421; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2422; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2423; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2424; GFX7-NEXT:    s_add_u32 s4, s0, 16
2425; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2426; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2427; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2428; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2429; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2430; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2431; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2432; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2433; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2434; GFX7-NEXT:    flat_store_dword v[0:1], v2
2435; GFX7-NEXT:    s_endpgm
2436;
2437; GFX10-WGP-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
2438; GFX10-WGP:       ; %bb.0: ; %entry
2439; GFX10-WGP-NEXT:    s_clause 0x1
2440; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2441; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2442; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2443; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2444; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2445; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2446; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2447; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2448; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2449; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2450; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2451; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2452; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2453; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2454; GFX10-WGP-NEXT:    s_endpgm
2455;
2456; GFX10-CU-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
2457; GFX10-CU:       ; %bb.0: ; %entry
2458; GFX10-CU-NEXT:    s_clause 0x1
2459; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2460; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2461; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2462; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2463; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
2464; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
2465; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2466; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
2467; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2468; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2469; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2470; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2471; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2472; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2473; GFX10-CU-NEXT:    s_endpgm
2474;
2475; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
2476; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2477; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2478; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2479; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2480; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
2481; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
2482; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2483; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2484; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2485; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2486; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2487; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2488; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2489; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2490; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2491; SKIP-CACHE-INV-NEXT:    s_endpgm
2492;
2493; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
2494; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2495; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2496; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2497; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2498; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2499; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2500; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2501; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2502; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2503; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2504;
2505; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
2506; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2507; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2508; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2509; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2510; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2511; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2512; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2513; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2514; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2515; GFX90A-TGSPLIT-NEXT:    s_endpgm
2516    i32* %out, i32 %in, i32 %old) {
2517entry:
2518  %gep = getelementptr i32, i32* %out, i32 4
2519  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic
2520  %val0 = extractvalue { i32, i1 } %val, 0
2521  store i32 %val0, i32* %out, align 4
2522  ret void
2523}
2524
2525define amdgpu_kernel void @flat_workgroup_acquire_monotonic_ret_cmpxchg(
2526; GFX7-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
2527; GFX7:       ; %bb.0: ; %entry
2528; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2529; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2530; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2531; GFX7-NEXT:    s_add_u32 s4, s0, 16
2532; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2533; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2534; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2535; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2536; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2537; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2538; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2539; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2540; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2541; GFX7-NEXT:    s_waitcnt vmcnt(0)
2542; GFX7-NEXT:    flat_store_dword v[0:1], v2
2543; GFX7-NEXT:    s_endpgm
2544;
2545; GFX10-WGP-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
2546; GFX10-WGP:       ; %bb.0: ; %entry
2547; GFX10-WGP-NEXT:    s_clause 0x1
2548; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2549; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2550; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2551; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2552; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2553; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2554; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2555; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2556; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2557; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2558; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2559; GFX10-WGP-NEXT:    buffer_gl0_inv
2560; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2561; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2562; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2563; GFX10-WGP-NEXT:    s_endpgm
2564;
2565; GFX10-CU-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
2566; GFX10-CU:       ; %bb.0: ; %entry
2567; GFX10-CU-NEXT:    s_clause 0x1
2568; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2569; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2570; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2571; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2572; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
2573; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
2574; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2575; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
2576; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2577; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2578; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2579; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2580; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2581; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
2582; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2583; GFX10-CU-NEXT:    s_endpgm
2584;
2585; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
2586; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2587; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2588; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2589; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2590; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
2591; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
2592; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2593; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2594; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2595; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2596; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2597; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2598; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2599; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2600; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2601; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2602; SKIP-CACHE-INV-NEXT:    s_endpgm
2603;
2604; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
2605; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2606; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2607; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2608; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2609; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2610; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2611; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2612; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2613; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2614; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2615;
2616; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
2617; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2618; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2619; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2620; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2621; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2622; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2623; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2624; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2625; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2626; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2627; GFX90A-TGSPLIT-NEXT:    s_endpgm
2628    i32* %out, i32 %in, i32 %old) {
2629entry:
2630  %gep = getelementptr i32, i32* %out, i32 4
2631  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
2632  %val0 = extractvalue { i32, i1 } %val, 0
2633  store i32 %val0, i32* %out, align 4
2634  ret void
2635}
2636
2637define amdgpu_kernel void @flat_workgroup_release_monotonic_ret_cmpxchg(
2638; GFX7-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
2639; GFX7:       ; %bb.0: ; %entry
2640; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2641; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2642; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2643; GFX7-NEXT:    s_add_u32 s4, s0, 16
2644; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2645; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2646; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2647; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2648; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2649; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2650; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2651; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2652; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2653; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2654; GFX7-NEXT:    flat_store_dword v[0:1], v2
2655; GFX7-NEXT:    s_endpgm
2656;
2657; GFX10-WGP-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
2658; GFX10-WGP:       ; %bb.0: ; %entry
2659; GFX10-WGP-NEXT:    s_clause 0x1
2660; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2661; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2662; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2663; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2664; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2665; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2666; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2667; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2668; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2669; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2670; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2671; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2672; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2673; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2674; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2675; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2676; GFX10-WGP-NEXT:    s_endpgm
2677;
2678; GFX10-CU-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
2679; GFX10-CU:       ; %bb.0: ; %entry
2680; GFX10-CU-NEXT:    s_clause 0x1
2681; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2682; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2683; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2684; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2685; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
2686; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
2687; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2688; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
2689; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2690; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2691; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2692; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2693; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2694; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2695; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2696; GFX10-CU-NEXT:    s_endpgm
2697;
2698; SKIP-CACHE-INV-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
2699; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2700; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2701; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2702; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2703; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
2704; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
2705; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2706; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2707; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2708; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2709; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2710; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2711; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2712; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2713; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2714; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2715; SKIP-CACHE-INV-NEXT:    s_endpgm
2716;
2717; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
2718; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2719; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2720; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2721; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2722; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2723; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2724; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2725; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2726; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2727; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2728; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2729;
2730; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
2731; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2732; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2733; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2734; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2735; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2736; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2737; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2738; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2739; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2740; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2741; GFX90A-TGSPLIT-NEXT:    s_endpgm
2742    i32* %out, i32 %in, i32 %old) {
2743entry:
2744  %gep = getelementptr i32, i32* %out, i32 4
2745  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic
2746  %val0 = extractvalue { i32, i1 } %val, 0
2747  store i32 %val0, i32* %out, align 4
2748  ret void
2749}
2750
2751define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_ret_cmpxchg(
2752; GFX7-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
2753; GFX7:       ; %bb.0: ; %entry
2754; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2755; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2756; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2757; GFX7-NEXT:    s_add_u32 s4, s0, 16
2758; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2759; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2760; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2761; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2762; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2763; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2764; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2765; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2766; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2767; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2768; GFX7-NEXT:    s_waitcnt vmcnt(0)
2769; GFX7-NEXT:    flat_store_dword v[0:1], v2
2770; GFX7-NEXT:    s_endpgm
2771;
2772; GFX10-WGP-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
2773; GFX10-WGP:       ; %bb.0: ; %entry
2774; GFX10-WGP-NEXT:    s_clause 0x1
2775; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2776; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2777; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2778; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2779; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2780; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2781; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2782; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2783; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2784; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2785; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2786; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2787; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2788; GFX10-WGP-NEXT:    buffer_gl0_inv
2789; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2790; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2791; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2792; GFX10-WGP-NEXT:    s_endpgm
2793;
2794; GFX10-CU-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
2795; GFX10-CU:       ; %bb.0: ; %entry
2796; GFX10-CU-NEXT:    s_clause 0x1
2797; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2798; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2799; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2800; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2801; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
2802; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
2803; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2804; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
2805; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2806; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2807; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2808; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2809; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2810; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2811; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
2812; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2813; GFX10-CU-NEXT:    s_endpgm
2814;
2815; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
2816; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2817; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2818; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2819; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2820; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
2821; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
2822; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2823; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2824; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2825; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2826; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2827; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2828; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2829; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2830; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2831; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2832; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2833; SKIP-CACHE-INV-NEXT:    s_endpgm
2834;
2835; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
2836; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2837; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2838; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2839; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2840; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2841; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2842; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2843; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2844; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2845; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2846; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2847;
2848; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
2849; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2850; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2851; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2852; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2853; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2854; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2855; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2856; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2857; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2858; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2859; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2860; GFX90A-TGSPLIT-NEXT:    s_endpgm
2861    i32* %out, i32 %in, i32 %old) {
2862entry:
2863  %gep = getelementptr i32, i32* %out, i32 4
2864  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
2865  %val0 = extractvalue { i32, i1 } %val, 0
2866  store i32 %val0, i32* %out, align 4
2867  ret void
2868}
2869
2870define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_ret_cmpxchg(
2871; GFX7-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
2872; GFX7:       ; %bb.0: ; %entry
2873; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2874; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2875; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2876; GFX7-NEXT:    s_add_u32 s4, s0, 16
2877; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2878; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2879; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2880; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2881; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2882; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2883; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2884; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2885; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2886; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2887; GFX7-NEXT:    s_waitcnt vmcnt(0)
2888; GFX7-NEXT:    flat_store_dword v[0:1], v2
2889; GFX7-NEXT:    s_endpgm
2890;
2891; GFX10-WGP-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
2892; GFX10-WGP:       ; %bb.0: ; %entry
2893; GFX10-WGP-NEXT:    s_clause 0x1
2894; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2895; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2896; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2897; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2898; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2899; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2900; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2901; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2902; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2903; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2904; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2905; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2906; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2907; GFX10-WGP-NEXT:    buffer_gl0_inv
2908; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2909; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2910; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2911; GFX10-WGP-NEXT:    s_endpgm
2912;
2913; GFX10-CU-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
2914; GFX10-CU:       ; %bb.0: ; %entry
2915; GFX10-CU-NEXT:    s_clause 0x1
2916; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2917; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2918; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2919; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2920; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
2921; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
2922; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2923; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
2924; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2925; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2926; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2927; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2928; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2929; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2930; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
2931; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2932; GFX10-CU-NEXT:    s_endpgm
2933;
2934; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
2935; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2936; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2937; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2938; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2939; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
2940; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
2941; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2942; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2943; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2944; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2945; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2946; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2947; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2948; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2949; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2950; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2951; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2952; SKIP-CACHE-INV-NEXT:    s_endpgm
2953;
2954; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
2955; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2956; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2957; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2958; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2959; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2960; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2961; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2962; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2963; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2964; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2965; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2966;
2967; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
2968; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2969; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2970; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2971; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2972; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2973; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2974; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2975; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2976; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2977; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2978; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2979; GFX90A-TGSPLIT-NEXT:    s_endpgm
2980    i32* %out, i32 %in, i32 %old) {
2981entry:
2982  %gep = getelementptr i32, i32* %out, i32 4
2983  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
2984  %val0 = extractvalue { i32, i1 } %val, 0
2985  store i32 %val0, i32* %out, align 4
2986  ret void
2987}
2988
2989define amdgpu_kernel void @flat_workgroup_monotonic_acquire_ret_cmpxchg(
2990; GFX7-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
2991; GFX7:       ; %bb.0: ; %entry
2992; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2993; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2994; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2995; GFX7-NEXT:    s_add_u32 s4, s0, 16
2996; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2997; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2998; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2999; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3000; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3001; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3002; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3003; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3004; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3005; GFX7-NEXT:    s_waitcnt vmcnt(0)
3006; GFX7-NEXT:    flat_store_dword v[0:1], v2
3007; GFX7-NEXT:    s_endpgm
3008;
3009; GFX10-WGP-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
3010; GFX10-WGP:       ; %bb.0: ; %entry
3011; GFX10-WGP-NEXT:    s_clause 0x1
3012; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3013; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3014; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3015; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3016; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3017; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3018; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3019; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3020; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3021; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3022; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3023; GFX10-WGP-NEXT:    buffer_gl0_inv
3024; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3025; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3026; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3027; GFX10-WGP-NEXT:    s_endpgm
3028;
3029; GFX10-CU-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
3030; GFX10-CU:       ; %bb.0: ; %entry
3031; GFX10-CU-NEXT:    s_clause 0x1
3032; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3033; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3034; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3035; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3036; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3037; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3038; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3039; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3040; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3041; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3042; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3043; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3044; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3045; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3046; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3047; GFX10-CU-NEXT:    s_endpgm
3048;
3049; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
3050; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3051; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3052; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3053; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3054; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3055; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3056; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3057; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3058; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3059; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3060; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3061; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3062; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3063; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3064; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3065; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3066; SKIP-CACHE-INV-NEXT:    s_endpgm
3067;
3068; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
3069; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3070; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3071; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3072; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3073; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3074; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3075; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3076; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3077; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3078; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3079;
3080; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
3081; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3082; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3083; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3084; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3085; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3086; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3087; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3088; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3089; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3090; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3091; GFX90A-TGSPLIT-NEXT:    s_endpgm
3092    i32* %out, i32 %in, i32 %old) {
3093entry:
3094  %gep = getelementptr i32, i32* %out, i32 4
3095  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic acquire
3096  %val0 = extractvalue { i32, i1 } %val, 0
3097  store i32 %val0, i32* %out, align 4
3098  ret void
3099}
3100
3101define amdgpu_kernel void @flat_workgroup_acquire_acquire_ret_cmpxchg(
3102; GFX7-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
3103; GFX7:       ; %bb.0: ; %entry
3104; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3105; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3106; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3107; GFX7-NEXT:    s_add_u32 s4, s0, 16
3108; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3109; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3110; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3111; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3112; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3113; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3114; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3115; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3116; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3117; GFX7-NEXT:    s_waitcnt vmcnt(0)
3118; GFX7-NEXT:    flat_store_dword v[0:1], v2
3119; GFX7-NEXT:    s_endpgm
3120;
3121; GFX10-WGP-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
3122; GFX10-WGP:       ; %bb.0: ; %entry
3123; GFX10-WGP-NEXT:    s_clause 0x1
3124; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3125; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3126; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3127; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3128; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3129; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3130; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3131; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3132; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3133; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3134; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3135; GFX10-WGP-NEXT:    buffer_gl0_inv
3136; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3137; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3138; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3139; GFX10-WGP-NEXT:    s_endpgm
3140;
3141; GFX10-CU-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
3142; GFX10-CU:       ; %bb.0: ; %entry
3143; GFX10-CU-NEXT:    s_clause 0x1
3144; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3145; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3146; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3147; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3148; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3149; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3150; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3151; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3152; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3153; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3154; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3155; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3156; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3157; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3158; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3159; GFX10-CU-NEXT:    s_endpgm
3160;
3161; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
3162; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3163; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3164; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3165; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3166; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3167; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3168; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3169; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3170; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3171; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3172; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3173; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3174; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3175; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3176; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3177; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3178; SKIP-CACHE-INV-NEXT:    s_endpgm
3179;
3180; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
3181; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3182; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3183; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3184; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3185; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3186; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3187; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3188; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3189; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3190; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3191;
3192; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
3193; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3194; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3195; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3196; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3197; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3198; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3199; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3200; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3201; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3202; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3203; GFX90A-TGSPLIT-NEXT:    s_endpgm
3204    i32* %out, i32 %in, i32 %old) {
3205entry:
3206  %gep = getelementptr i32, i32* %out, i32 4
3207  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
3208  %val0 = extractvalue { i32, i1 } %val, 0
3209  store i32 %val0, i32* %out, align 4
3210  ret void
3211}
3212
3213define amdgpu_kernel void @flat_workgroup_release_acquire_ret_cmpxchg(
3214; GFX7-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
3215; GFX7:       ; %bb.0: ; %entry
3216; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3217; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3218; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3219; GFX7-NEXT:    s_add_u32 s4, s0, 16
3220; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3221; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3222; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3223; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3224; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3225; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3226; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3227; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3228; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3229; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3230; GFX7-NEXT:    s_waitcnt vmcnt(0)
3231; GFX7-NEXT:    flat_store_dword v[0:1], v2
3232; GFX7-NEXT:    s_endpgm
3233;
3234; GFX10-WGP-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
3235; GFX10-WGP:       ; %bb.0: ; %entry
3236; GFX10-WGP-NEXT:    s_clause 0x1
3237; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3238; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3239; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3240; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3241; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3242; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3243; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3244; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3245; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3246; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3247; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3248; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3249; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3250; GFX10-WGP-NEXT:    buffer_gl0_inv
3251; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3252; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3253; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3254; GFX10-WGP-NEXT:    s_endpgm
3255;
3256; GFX10-CU-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
3257; GFX10-CU:       ; %bb.0: ; %entry
3258; GFX10-CU-NEXT:    s_clause 0x1
3259; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3260; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3261; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3262; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3263; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3264; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3265; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3266; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3267; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3268; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3269; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3270; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3271; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3272; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3273; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3274; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3275; GFX10-CU-NEXT:    s_endpgm
3276;
3277; SKIP-CACHE-INV-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
3278; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3279; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3280; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3281; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3282; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3283; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3284; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3285; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3286; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3287; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3288; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3289; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3290; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3291; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3292; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3293; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3294; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3295; SKIP-CACHE-INV-NEXT:    s_endpgm
3296;
3297; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
3298; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3299; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3300; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3301; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3302; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3303; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3304; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3305; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3306; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3307; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3308; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3309;
3310; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
3311; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3312; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3313; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3314; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3315; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3316; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3317; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3318; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3319; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3320; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3321; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3322; GFX90A-TGSPLIT-NEXT:    s_endpgm
3323    i32* %out, i32 %in, i32 %old) {
3324entry:
3325  %gep = getelementptr i32, i32* %out, i32 4
3326  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
3327  %val0 = extractvalue { i32, i1 } %val, 0
3328  store i32 %val0, i32* %out, align 4
3329  ret void
3330}
3331
3332define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_ret_cmpxchg(
3333; GFX7-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
3334; GFX7:       ; %bb.0: ; %entry
3335; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3336; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3337; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3338; GFX7-NEXT:    s_add_u32 s4, s0, 16
3339; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3340; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3341; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3342; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3343; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3344; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3345; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3346; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3347; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3348; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3349; GFX7-NEXT:    s_waitcnt vmcnt(0)
3350; GFX7-NEXT:    flat_store_dword v[0:1], v2
3351; GFX7-NEXT:    s_endpgm
3352;
3353; GFX10-WGP-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
3354; GFX10-WGP:       ; %bb.0: ; %entry
3355; GFX10-WGP-NEXT:    s_clause 0x1
3356; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3357; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3358; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3359; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3360; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3361; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3362; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3363; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3364; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3365; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3366; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3367; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3368; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3369; GFX10-WGP-NEXT:    buffer_gl0_inv
3370; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3371; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3372; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3373; GFX10-WGP-NEXT:    s_endpgm
3374;
3375; GFX10-CU-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
3376; GFX10-CU:       ; %bb.0: ; %entry
3377; GFX10-CU-NEXT:    s_clause 0x1
3378; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3379; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3380; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3381; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3382; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3383; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3384; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3385; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3386; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3387; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3388; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3389; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3390; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3391; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3392; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3393; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3394; GFX10-CU-NEXT:    s_endpgm
3395;
3396; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
3397; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3398; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3399; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3400; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3401; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3402; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3403; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3404; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3405; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3406; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3407; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3408; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3409; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3410; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3411; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3412; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3413; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3414; SKIP-CACHE-INV-NEXT:    s_endpgm
3415;
3416; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
3417; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3418; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3419; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3420; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3421; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3422; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3423; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3424; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3425; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3426; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3427; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3428;
3429; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
3430; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3431; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3432; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3433; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3434; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3435; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3436; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3437; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3438; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3439; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3440; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3441; GFX90A-TGSPLIT-NEXT:    s_endpgm
3442    i32* %out, i32 %in, i32 %old) {
3443entry:
3444  %gep = getelementptr i32, i32* %out, i32 4
3445  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
3446  %val0 = extractvalue { i32, i1 } %val, 0
3447  store i32 %val0, i32* %out, align 4
3448  ret void
3449}
3450
3451define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_ret_cmpxchg(
3452; GFX7-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
3453; GFX7:       ; %bb.0: ; %entry
3454; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3455; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3456; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3457; GFX7-NEXT:    s_add_u32 s4, s0, 16
3458; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3459; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3460; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3461; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3462; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3463; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3464; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3465; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3466; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3467; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3468; GFX7-NEXT:    s_waitcnt vmcnt(0)
3469; GFX7-NEXT:    flat_store_dword v[0:1], v2
3470; GFX7-NEXT:    s_endpgm
3471;
3472; GFX10-WGP-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
3473; GFX10-WGP:       ; %bb.0: ; %entry
3474; GFX10-WGP-NEXT:    s_clause 0x1
3475; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3476; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3477; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3478; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3479; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3480; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3481; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3482; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3483; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3484; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3485; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3486; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3487; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3488; GFX10-WGP-NEXT:    buffer_gl0_inv
3489; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3490; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3491; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3492; GFX10-WGP-NEXT:    s_endpgm
3493;
3494; GFX10-CU-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
3495; GFX10-CU:       ; %bb.0: ; %entry
3496; GFX10-CU-NEXT:    s_clause 0x1
3497; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3498; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3499; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3500; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3501; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3502; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3503; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3504; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3505; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3506; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3507; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3508; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3509; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3510; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3511; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3512; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3513; GFX10-CU-NEXT:    s_endpgm
3514;
3515; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
3516; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3517; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3518; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3519; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3520; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3521; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3522; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3523; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3524; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3525; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3526; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3527; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3528; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3529; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3530; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3531; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3532; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3533; SKIP-CACHE-INV-NEXT:    s_endpgm
3534;
3535; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
3536; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3537; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3538; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3539; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3540; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3541; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3542; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3543; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3544; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3545; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3546; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3547;
3548; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
3549; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3550; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3551; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3552; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3553; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3554; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3555; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3556; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3557; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3558; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3559; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3560; GFX90A-TGSPLIT-NEXT:    s_endpgm
3561    i32* %out, i32 %in, i32 %old) {
3562entry:
3563  %gep = getelementptr i32, i32* %out, i32 4
3564  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
3565  %val0 = extractvalue { i32, i1 } %val, 0
3566  store i32 %val0, i32* %out, align 4
3567  ret void
3568}
3569
3570define amdgpu_kernel void @flat_workgroup_monotonic_seq_cst_ret_cmpxchg(
3571; GFX7-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
3572; GFX7:       ; %bb.0: ; %entry
3573; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3574; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3575; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3576; GFX7-NEXT:    s_add_u32 s4, s0, 16
3577; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3578; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3579; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3580; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3581; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3582; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3583; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3584; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3585; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3586; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3587; GFX7-NEXT:    s_waitcnt vmcnt(0)
3588; GFX7-NEXT:    flat_store_dword v[0:1], v2
3589; GFX7-NEXT:    s_endpgm
3590;
3591; GFX10-WGP-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
3592; GFX10-WGP:       ; %bb.0: ; %entry
3593; GFX10-WGP-NEXT:    s_clause 0x1
3594; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3595; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3596; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3597; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3598; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3599; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3600; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3601; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3602; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3603; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3604; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3605; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3606; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3607; GFX10-WGP-NEXT:    buffer_gl0_inv
3608; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3609; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3610; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3611; GFX10-WGP-NEXT:    s_endpgm
3612;
3613; GFX10-CU-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
3614; GFX10-CU:       ; %bb.0: ; %entry
3615; GFX10-CU-NEXT:    s_clause 0x1
3616; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3617; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3618; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3619; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3620; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3621; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3622; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3623; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3624; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3625; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3626; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3627; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3628; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3629; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3630; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3631; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3632; GFX10-CU-NEXT:    s_endpgm
3633;
3634; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
3635; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3636; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3637; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3638; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3639; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3640; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3641; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3642; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3643; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3644; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3645; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3646; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3647; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3648; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3649; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3650; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3651; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3652; SKIP-CACHE-INV-NEXT:    s_endpgm
3653;
3654; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
3655; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3656; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3657; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3658; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3659; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3660; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3661; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3662; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3663; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3664; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3665; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3666;
3667; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
3668; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3669; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3670; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3671; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3672; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3673; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3674; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3675; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3676; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3677; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3678; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3679; GFX90A-TGSPLIT-NEXT:    s_endpgm
3680    i32* %out, i32 %in, i32 %old) {
3681entry:
3682  %gep = getelementptr i32, i32* %out, i32 4
3683  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic seq_cst
3684  %val0 = extractvalue { i32, i1 } %val, 0
3685  store i32 %val0, i32* %out, align 4
3686  ret void
3687}
3688
3689define amdgpu_kernel void @flat_workgroup_acquire_seq_cst_ret_cmpxchg(
3690; GFX7-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
3691; GFX7:       ; %bb.0: ; %entry
3692; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3693; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3694; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3695; GFX7-NEXT:    s_add_u32 s4, s0, 16
3696; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3697; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3698; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3699; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3700; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3701; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3702; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3703; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3704; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3705; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3706; GFX7-NEXT:    s_waitcnt vmcnt(0)
3707; GFX7-NEXT:    flat_store_dword v[0:1], v2
3708; GFX7-NEXT:    s_endpgm
3709;
3710; GFX10-WGP-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
3711; GFX10-WGP:       ; %bb.0: ; %entry
3712; GFX10-WGP-NEXT:    s_clause 0x1
3713; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3714; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3715; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3716; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3717; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3718; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3719; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3720; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3721; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3722; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3723; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3724; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3725; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3726; GFX10-WGP-NEXT:    buffer_gl0_inv
3727; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3728; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3729; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3730; GFX10-WGP-NEXT:    s_endpgm
3731;
3732; GFX10-CU-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
3733; GFX10-CU:       ; %bb.0: ; %entry
3734; GFX10-CU-NEXT:    s_clause 0x1
3735; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3736; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3737; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3738; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3739; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3740; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3741; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3742; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3743; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3744; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3745; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3746; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3747; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3748; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3749; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3750; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3751; GFX10-CU-NEXT:    s_endpgm
3752;
3753; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
3754; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3755; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3756; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3757; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3758; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3759; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3760; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3761; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3762; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3763; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3764; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3765; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3766; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3767; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3768; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3769; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3770; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3771; SKIP-CACHE-INV-NEXT:    s_endpgm
3772;
3773; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
3774; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3775; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3776; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3777; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3778; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3779; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3780; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3781; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3782; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3783; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3784; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3785;
3786; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
3787; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3788; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3789; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3790; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3791; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3792; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3793; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3794; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3795; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3796; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3797; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3798; GFX90A-TGSPLIT-NEXT:    s_endpgm
3799    i32* %out, i32 %in, i32 %old) {
3800entry:
3801  %gep = getelementptr i32, i32* %out, i32 4
3802  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire seq_cst
3803  %val0 = extractvalue { i32, i1 } %val, 0
3804  store i32 %val0, i32* %out, align 4
3805  ret void
3806}
3807
3808define amdgpu_kernel void @flat_workgroup_release_seq_cst_ret_cmpxchg(
3809; GFX7-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
3810; GFX7:       ; %bb.0: ; %entry
3811; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3812; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3813; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3814; GFX7-NEXT:    s_add_u32 s4, s0, 16
3815; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3816; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3817; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3818; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3819; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3820; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3821; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3822; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3823; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3824; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3825; GFX7-NEXT:    s_waitcnt vmcnt(0)
3826; GFX7-NEXT:    flat_store_dword v[0:1], v2
3827; GFX7-NEXT:    s_endpgm
3828;
3829; GFX10-WGP-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
3830; GFX10-WGP:       ; %bb.0: ; %entry
3831; GFX10-WGP-NEXT:    s_clause 0x1
3832; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3833; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3834; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3835; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3836; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3837; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3838; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3839; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3840; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3841; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3842; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3843; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3844; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3845; GFX10-WGP-NEXT:    buffer_gl0_inv
3846; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3847; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3848; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3849; GFX10-WGP-NEXT:    s_endpgm
3850;
3851; GFX10-CU-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
3852; GFX10-CU:       ; %bb.0: ; %entry
3853; GFX10-CU-NEXT:    s_clause 0x1
3854; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3855; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3856; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3857; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3858; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3859; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3860; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3861; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3862; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3863; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3864; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3865; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3866; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3867; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3868; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3869; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3870; GFX10-CU-NEXT:    s_endpgm
3871;
3872; SKIP-CACHE-INV-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
3873; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3874; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3875; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3876; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3877; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3878; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3879; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3880; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3881; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3882; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3883; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3884; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3885; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3886; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3887; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3888; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3889; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3890; SKIP-CACHE-INV-NEXT:    s_endpgm
3891;
3892; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
3893; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3894; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3895; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3896; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3897; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3898; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3899; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3900; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3901; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3902; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3903; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3904;
3905; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
3906; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3907; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3908; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3909; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3910; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3911; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3912; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3913; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3914; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3915; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3916; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3917; GFX90A-TGSPLIT-NEXT:    s_endpgm
3918    i32* %out, i32 %in, i32 %old) {
3919entry:
3920  %gep = getelementptr i32, i32* %out, i32 4
3921  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release seq_cst
3922  %val0 = extractvalue { i32, i1 } %val, 0
3923  store i32 %val0, i32* %out, align 4
3924  ret void
3925}
3926
3927define amdgpu_kernel void @flat_workgroup_acq_rel_seq_cst_ret_cmpxchg(
3928; GFX7-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
3929; GFX7:       ; %bb.0: ; %entry
3930; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3931; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3932; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3933; GFX7-NEXT:    s_add_u32 s4, s0, 16
3934; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3935; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3936; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3937; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3938; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3939; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3940; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3941; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3942; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3943; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3944; GFX7-NEXT:    s_waitcnt vmcnt(0)
3945; GFX7-NEXT:    flat_store_dword v[0:1], v2
3946; GFX7-NEXT:    s_endpgm
3947;
3948; GFX10-WGP-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
3949; GFX10-WGP:       ; %bb.0: ; %entry
3950; GFX10-WGP-NEXT:    s_clause 0x1
3951; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3952; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3953; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3954; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3955; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3956; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3957; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3958; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3959; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3960; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3961; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3962; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3963; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3964; GFX10-WGP-NEXT:    buffer_gl0_inv
3965; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3966; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3967; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3968; GFX10-WGP-NEXT:    s_endpgm
3969;
3970; GFX10-CU-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
3971; GFX10-CU:       ; %bb.0: ; %entry
3972; GFX10-CU-NEXT:    s_clause 0x1
3973; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3974; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3975; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3976; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3977; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3978; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3979; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3980; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3981; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3982; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3983; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3984; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3985; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3986; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3987; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3988; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3989; GFX10-CU-NEXT:    s_endpgm
3990;
3991; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
3992; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3993; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3994; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3995; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3996; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3997; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3998; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3999; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4000; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4001; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4002; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4003; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4004; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4005; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4006; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4007; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4008; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4009; SKIP-CACHE-INV-NEXT:    s_endpgm
4010;
4011; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
4012; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4013; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4014; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4015; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4016; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4017; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4018; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4019; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4020; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4021; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4022; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4023;
4024; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
4025; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4026; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4027; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4028; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4029; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4030; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4031; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4032; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4033; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4034; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4035; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4036; GFX90A-TGSPLIT-NEXT:    s_endpgm
4037    i32* %out, i32 %in, i32 %old) {
4038entry:
4039  %gep = getelementptr i32, i32* %out, i32 4
4040  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel seq_cst
4041  %val0 = extractvalue { i32, i1 } %val, 0
4042  store i32 %val0, i32* %out, align 4
4043  ret void
4044}
4045
4046define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_ret_cmpxchg(
4047; GFX7-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
4048; GFX7:       ; %bb.0: ; %entry
4049; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4050; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4051; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4052; GFX7-NEXT:    s_add_u32 s4, s0, 16
4053; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4054; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4055; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4056; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4057; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4058; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4059; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4060; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4061; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4062; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4063; GFX7-NEXT:    s_waitcnt vmcnt(0)
4064; GFX7-NEXT:    flat_store_dword v[0:1], v2
4065; GFX7-NEXT:    s_endpgm
4066;
4067; GFX10-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
4068; GFX10-WGP:       ; %bb.0: ; %entry
4069; GFX10-WGP-NEXT:    s_clause 0x1
4070; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4071; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4072; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4073; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4074; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4075; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4076; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4077; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4078; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4079; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4080; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4081; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4082; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4083; GFX10-WGP-NEXT:    buffer_gl0_inv
4084; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4085; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4086; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4087; GFX10-WGP-NEXT:    s_endpgm
4088;
4089; GFX10-CU-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
4090; GFX10-CU:       ; %bb.0: ; %entry
4091; GFX10-CU-NEXT:    s_clause 0x1
4092; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4093; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4094; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4095; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4096; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4097; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4098; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4099; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4100; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4101; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4102; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4103; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4104; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4105; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4106; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4107; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4108; GFX10-CU-NEXT:    s_endpgm
4109;
4110; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
4111; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4112; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4113; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4114; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4115; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4116; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4117; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4118; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4119; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4120; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4121; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4122; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4123; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4124; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4125; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4126; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4127; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4128; SKIP-CACHE-INV-NEXT:    s_endpgm
4129;
4130; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
4131; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4132; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4133; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4134; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4135; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4136; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4137; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4138; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4139; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4140; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4141; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4142;
4143; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
4144; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4145; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4146; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4147; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4148; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4149; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4150; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4151; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4152; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4153; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4154; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4155; GFX90A-TGSPLIT-NEXT:    s_endpgm
4156    i32* %out, i32 %in, i32 %old) {
4157entry:
4158  %gep = getelementptr i32, i32* %out, i32 4
4159  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
4160  %val0 = extractvalue { i32, i1 } %val, 0
4161  store i32 %val0, i32* %out, align 4
4162  ret void
4163}
4164
4165define amdgpu_kernel void @flat_workgroup_one_as_unordered_load(
4166; GFX7-LABEL: flat_workgroup_one_as_unordered_load:
4167; GFX7:       ; %bb.0: ; %entry
4168; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4169; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4170; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4171; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4172; GFX7-NEXT:    flat_load_dword v2, v[0:1]
4173; GFX7-NEXT:    v_mov_b32_e32 v0, s2
4174; GFX7-NEXT:    v_mov_b32_e32 v1, s3
4175; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4176; GFX7-NEXT:    flat_store_dword v[0:1], v2
4177; GFX7-NEXT:    s_endpgm
4178;
4179; GFX10-WGP-LABEL: flat_workgroup_one_as_unordered_load:
4180; GFX10-WGP:       ; %bb.0: ; %entry
4181; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4182; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4183; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4184; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4185; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
4186; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
4187; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
4188; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4189; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4190; GFX10-WGP-NEXT:    s_endpgm
4191;
4192; GFX10-CU-LABEL: flat_workgroup_one_as_unordered_load:
4193; GFX10-CU:       ; %bb.0: ; %entry
4194; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4195; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4196; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4197; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4198; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
4199; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
4200; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
4201; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4202; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4203; GFX10-CU-NEXT:    s_endpgm
4204;
4205; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_unordered_load:
4206; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4207; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4208; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4209; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4210; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4211; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
4212; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4213; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4214; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4215; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4216; SKIP-CACHE-INV-NEXT:    s_endpgm
4217;
4218; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_load:
4219; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4220; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4221; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4222; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4223; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4224; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
4225; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
4226; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
4227; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4228; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4229; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4230;
4231; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_load:
4232; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4233; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4234; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4235; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4236; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4237; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
4238; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
4239; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
4240; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4241; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4242; GFX90A-TGSPLIT-NEXT:    s_endpgm
4243    i32* %in, i32* %out) {
4244entry:
4245  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") unordered, align 4
4246  store i32 %val, i32* %out
4247  ret void
4248}
4249
4250define amdgpu_kernel void @flat_workgroup_one_as_monotonic_load(
4251; GFX7-LABEL: flat_workgroup_one_as_monotonic_load:
4252; GFX7:       ; %bb.0: ; %entry
4253; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4254; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4255; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4256; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4257; GFX7-NEXT:    flat_load_dword v2, v[0:1]
4258; GFX7-NEXT:    v_mov_b32_e32 v0, s2
4259; GFX7-NEXT:    v_mov_b32_e32 v1, s3
4260; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4261; GFX7-NEXT:    flat_store_dword v[0:1], v2
4262; GFX7-NEXT:    s_endpgm
4263;
4264; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_load:
4265; GFX10-WGP:       ; %bb.0: ; %entry
4266; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4267; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4268; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4269; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4270; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
4271; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
4272; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
4273; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4274; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4275; GFX10-WGP-NEXT:    s_endpgm
4276;
4277; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_load:
4278; GFX10-CU:       ; %bb.0: ; %entry
4279; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4280; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4281; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4282; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4283; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
4284; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
4285; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
4286; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4287; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4288; GFX10-CU-NEXT:    s_endpgm
4289;
4290; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_load:
4291; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4292; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4293; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4294; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4295; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4296; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
4297; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4298; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4299; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4300; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4301; SKIP-CACHE-INV-NEXT:    s_endpgm
4302;
4303; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load:
4304; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4305; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4306; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4307; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4308; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4309; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
4310; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
4311; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
4312; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4313; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4314; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4315;
4316; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load:
4317; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4318; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4319; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4320; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4321; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4322; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
4323; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
4324; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
4325; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4326; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4327; GFX90A-TGSPLIT-NEXT:    s_endpgm
4328    i32* %in, i32* %out) {
4329entry:
4330  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") monotonic, align 4
4331  store i32 %val, i32* %out
4332  ret void
4333}
4334
4335define amdgpu_kernel void @flat_workgroup_one_as_acquire_load(
4336; GFX7-LABEL: flat_workgroup_one_as_acquire_load:
4337; GFX7:       ; %bb.0: ; %entry
4338; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4339; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4340; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4341; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4342; GFX7-NEXT:    flat_load_dword v2, v[0:1]
4343; GFX7-NEXT:    v_mov_b32_e32 v0, s2
4344; GFX7-NEXT:    v_mov_b32_e32 v1, s3
4345; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4346; GFX7-NEXT:    flat_store_dword v[0:1], v2
4347; GFX7-NEXT:    s_endpgm
4348;
4349; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_load:
4350; GFX10-WGP:       ; %bb.0: ; %entry
4351; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4352; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4353; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4354; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4355; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
4356; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4357; GFX10-WGP-NEXT:    buffer_gl0_inv
4358; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
4359; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
4360; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4361; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4362; GFX10-WGP-NEXT:    s_endpgm
4363;
4364; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_load:
4365; GFX10-CU:       ; %bb.0: ; %entry
4366; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4367; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4368; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4369; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4370; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
4371; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
4372; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
4373; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4374; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4375; GFX10-CU-NEXT:    s_endpgm
4376;
4377; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_load:
4378; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4379; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4380; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4381; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4382; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4383; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
4384; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4385; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4386; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4387; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4388; SKIP-CACHE-INV-NEXT:    s_endpgm
4389;
4390; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_load:
4391; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4392; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4393; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4394; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4395; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4396; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
4397; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
4398; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
4399; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4400; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4401; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4402;
4403; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_load:
4404; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4405; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4406; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4407; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4408; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4409; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
4410; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4411; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4412; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
4413; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
4414; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4415; GFX90A-TGSPLIT-NEXT:    s_endpgm
4416    i32* %in, i32* %out) {
4417entry:
4418  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") acquire, align 4
4419  store i32 %val, i32* %out
4420  ret void
4421}
4422
4423define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_load(
4424; GFX7-LABEL: flat_workgroup_one_as_seq_cst_load:
4425; GFX7:       ; %bb.0: ; %entry
4426; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4427; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4428; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4429; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4430; GFX7-NEXT:    flat_load_dword v2, v[0:1]
4431; GFX7-NEXT:    v_mov_b32_e32 v0, s2
4432; GFX7-NEXT:    v_mov_b32_e32 v1, s3
4433; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4434; GFX7-NEXT:    flat_store_dword v[0:1], v2
4435; GFX7-NEXT:    s_endpgm
4436;
4437; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_load:
4438; GFX10-WGP:       ; %bb.0: ; %entry
4439; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4440; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4441; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4442; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4443; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4444; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4445; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
4446; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4447; GFX10-WGP-NEXT:    buffer_gl0_inv
4448; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
4449; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
4450; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4451; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4452; GFX10-WGP-NEXT:    s_endpgm
4453;
4454; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_load:
4455; GFX10-CU:       ; %bb.0: ; %entry
4456; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4457; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4458; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4459; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4460; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
4461; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
4462; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
4463; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4464; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4465; GFX10-CU-NEXT:    s_endpgm
4466;
4467; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_load:
4468; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4469; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4470; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4471; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4472; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4473; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
4474; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4475; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4476; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4477; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4478; SKIP-CACHE-INV-NEXT:    s_endpgm
4479;
4480; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load:
4481; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4482; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4483; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4484; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4485; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4486; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
4487; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
4488; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
4489; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4490; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4491; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4492;
4493; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load:
4494; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4495; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4496; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4497; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4498; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4499; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4500; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
4501; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4502; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4503; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
4504; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
4505; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4506; GFX90A-TGSPLIT-NEXT:    s_endpgm
4507    i32* %in, i32* %out) {
4508entry:
4509  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") seq_cst, align 4
4510  store i32 %val, i32* %out
4511  ret void
4512}
4513
4514define amdgpu_kernel void @flat_workgroup_one_as_unordered_store(
4515; GFX7-LABEL: flat_workgroup_one_as_unordered_store:
4516; GFX7:       ; %bb.0: ; %entry
4517; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
4518; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
4519; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4520; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4521; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4522; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4523; GFX7-NEXT:    flat_store_dword v[0:1], v2
4524; GFX7-NEXT:    s_endpgm
4525;
4526; GFX10-WGP-LABEL: flat_workgroup_one_as_unordered_store:
4527; GFX10-WGP:       ; %bb.0: ; %entry
4528; GFX10-WGP-NEXT:    s_clause 0x1
4529; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4530; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
4531; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4532; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4533; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4534; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4535; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4536; GFX10-WGP-NEXT:    s_endpgm
4537;
4538; GFX10-CU-LABEL: flat_workgroup_one_as_unordered_store:
4539; GFX10-CU:       ; %bb.0: ; %entry
4540; GFX10-CU-NEXT:    s_clause 0x1
4541; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4542; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
4543; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4544; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4545; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4546; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4547; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4548; GFX10-CU-NEXT:    s_endpgm
4549;
4550; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_unordered_store:
4551; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4552; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
4553; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
4554; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4555; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4556; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4557; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4558; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4559; SKIP-CACHE-INV-NEXT:    s_endpgm
4560;
4561; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_store:
4562; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4563; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4564; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4565; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4566; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4567; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4568; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4569; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4570;
4571; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_store:
4572; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4573; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4574; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4575; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4576; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4577; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4578; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4579; GFX90A-TGSPLIT-NEXT:    s_endpgm
4580    i32 %in, i32* %out) {
4581entry:
4582  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") unordered, align 4
4583  ret void
4584}
4585
4586define amdgpu_kernel void @flat_workgroup_one_as_monotonic_store(
4587; GFX7-LABEL: flat_workgroup_one_as_monotonic_store:
4588; GFX7:       ; %bb.0: ; %entry
4589; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
4590; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
4591; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4592; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4593; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4594; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4595; GFX7-NEXT:    flat_store_dword v[0:1], v2
4596; GFX7-NEXT:    s_endpgm
4597;
4598; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_store:
4599; GFX10-WGP:       ; %bb.0: ; %entry
4600; GFX10-WGP-NEXT:    s_clause 0x1
4601; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4602; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
4603; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4604; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4605; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4606; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4607; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4608; GFX10-WGP-NEXT:    s_endpgm
4609;
4610; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_store:
4611; GFX10-CU:       ; %bb.0: ; %entry
4612; GFX10-CU-NEXT:    s_clause 0x1
4613; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4614; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
4615; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4616; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4617; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4618; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4619; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4620; GFX10-CU-NEXT:    s_endpgm
4621;
4622; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_store:
4623; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4624; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
4625; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
4626; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4627; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4628; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4629; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4630; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4631; SKIP-CACHE-INV-NEXT:    s_endpgm
4632;
4633; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store:
4634; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4635; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4636; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4637; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4638; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4639; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4640; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4641; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4642;
4643; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store:
4644; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4645; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4646; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4647; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4648; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4649; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4650; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4651; GFX90A-TGSPLIT-NEXT:    s_endpgm
4652    i32 %in, i32* %out) {
4653entry:
4654  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") monotonic, align 4
4655  ret void
4656}
4657
4658define amdgpu_kernel void @flat_workgroup_one_as_release_store(
4659; GFX7-LABEL: flat_workgroup_one_as_release_store:
4660; GFX7:       ; %bb.0: ; %entry
4661; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
4662; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
4663; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4664; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4665; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4666; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4667; GFX7-NEXT:    flat_store_dword v[0:1], v2
4668; GFX7-NEXT:    s_endpgm
4669;
4670; GFX10-WGP-LABEL: flat_workgroup_one_as_release_store:
4671; GFX10-WGP:       ; %bb.0: ; %entry
4672; GFX10-WGP-NEXT:    s_clause 0x1
4673; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4674; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
4675; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4676; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4677; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4678; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4679; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4680; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4681; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4682; GFX10-WGP-NEXT:    s_endpgm
4683;
4684; GFX10-CU-LABEL: flat_workgroup_one_as_release_store:
4685; GFX10-CU:       ; %bb.0: ; %entry
4686; GFX10-CU-NEXT:    s_clause 0x1
4687; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4688; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
4689; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4690; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4691; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4692; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4693; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4694; GFX10-CU-NEXT:    s_endpgm
4695;
4696; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_store:
4697; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4698; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
4699; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
4700; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4701; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4702; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4703; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4704; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4705; SKIP-CACHE-INV-NEXT:    s_endpgm
4706;
4707; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_store:
4708; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4709; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4710; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4711; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4712; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4713; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4714; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4715; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4716;
4717; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_store:
4718; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4719; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4720; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4721; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4722; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4723; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4724; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4725; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4726; GFX90A-TGSPLIT-NEXT:    s_endpgm
4727    i32 %in, i32* %out) {
4728entry:
4729  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") release, align 4
4730  ret void
4731}
4732
4733define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_store(
4734; GFX7-LABEL: flat_workgroup_one_as_seq_cst_store:
4735; GFX7:       ; %bb.0: ; %entry
4736; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
4737; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
4738; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4739; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4740; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4741; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4742; GFX7-NEXT:    flat_store_dword v[0:1], v2
4743; GFX7-NEXT:    s_endpgm
4744;
4745; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_store:
4746; GFX10-WGP:       ; %bb.0: ; %entry
4747; GFX10-WGP-NEXT:    s_clause 0x1
4748; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4749; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
4750; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4751; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4752; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4753; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4754; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4755; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4756; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4757; GFX10-WGP-NEXT:    s_endpgm
4758;
4759; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_store:
4760; GFX10-CU:       ; %bb.0: ; %entry
4761; GFX10-CU-NEXT:    s_clause 0x1
4762; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4763; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
4764; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4765; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4766; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4767; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4768; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4769; GFX10-CU-NEXT:    s_endpgm
4770;
4771; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_store:
4772; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4773; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
4774; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
4775; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4776; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4777; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4778; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4779; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4780; SKIP-CACHE-INV-NEXT:    s_endpgm
4781;
4782; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store:
4783; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4784; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4785; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4786; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4787; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4788; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4789; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4790; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4791;
4792; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store:
4793; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4794; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4795; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4796; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4797; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4798; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4799; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4800; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4801; GFX90A-TGSPLIT-NEXT:    s_endpgm
4802    i32 %in, i32* %out) {
4803entry:
4804  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") seq_cst, align 4
4805  ret void
4806}
4807
4808define amdgpu_kernel void @flat_workgroup_one_as_monotonic_atomicrmw(
4809; GFX7-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
4810; GFX7:       ; %bb.0: ; %entry
4811; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4812; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4813; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4814; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4815; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4816; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4817; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4818; GFX7-NEXT:    s_endpgm
4819;
4820; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
4821; GFX10-WGP:       ; %bb.0: ; %entry
4822; GFX10-WGP-NEXT:    s_clause 0x1
4823; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4824; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4825; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4826; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4827; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4828; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4829; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
4830; GFX10-WGP-NEXT:    s_endpgm
4831;
4832; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
4833; GFX10-CU:       ; %bb.0: ; %entry
4834; GFX10-CU-NEXT:    s_clause 0x1
4835; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4836; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4837; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4838; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4839; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4840; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4841; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
4842; GFX10-CU-NEXT:    s_endpgm
4843;
4844; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
4845; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4846; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4847; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
4848; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4849; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4850; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4851; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4852; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
4853; SKIP-CACHE-INV-NEXT:    s_endpgm
4854;
4855; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
4856; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4857; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4858; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4859; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4860; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4861; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4862; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4863; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4864;
4865; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
4866; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4867; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4868; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4869; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4870; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4871; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4872; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4873; GFX90A-TGSPLIT-NEXT:    s_endpgm
4874    i32* %out, i32 %in) {
4875entry:
4876  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") monotonic
4877  ret void
4878}
4879
4880define amdgpu_kernel void @flat_workgroup_one_as_acquire_atomicrmw(
4881; GFX7-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
4882; GFX7:       ; %bb.0: ; %entry
4883; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4884; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4885; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4886; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4887; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4888; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4889; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4890; GFX7-NEXT:    s_endpgm
4891;
4892; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
4893; GFX10-WGP:       ; %bb.0: ; %entry
4894; GFX10-WGP-NEXT:    s_clause 0x1
4895; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4896; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4897; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4898; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4899; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4900; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4901; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
4902; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4903; GFX10-WGP-NEXT:    buffer_gl0_inv
4904; GFX10-WGP-NEXT:    s_endpgm
4905;
4906; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
4907; GFX10-CU:       ; %bb.0: ; %entry
4908; GFX10-CU-NEXT:    s_clause 0x1
4909; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4910; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4911; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4912; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4913; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4914; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4915; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
4916; GFX10-CU-NEXT:    s_endpgm
4917;
4918; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
4919; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4920; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4921; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
4922; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4923; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4924; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4925; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4926; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
4927; SKIP-CACHE-INV-NEXT:    s_endpgm
4928;
4929; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
4930; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4931; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4932; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4933; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4934; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4935; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4936; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4937; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4938;
4939; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
4940; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4941; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4942; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4943; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4944; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4945; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4946; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4947; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4948; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4949; GFX90A-TGSPLIT-NEXT:    s_endpgm
4950    i32* %out, i32 %in) {
4951entry:
4952  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acquire
4953  ret void
4954}
4955
4956define amdgpu_kernel void @flat_workgroup_one_as_release_atomicrmw(
4957; GFX7-LABEL: flat_workgroup_one_as_release_atomicrmw:
4958; GFX7:       ; %bb.0: ; %entry
4959; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4960; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4961; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4962; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4963; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4964; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4965; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4966; GFX7-NEXT:    s_endpgm
4967;
4968; GFX10-WGP-LABEL: flat_workgroup_one_as_release_atomicrmw:
4969; GFX10-WGP:       ; %bb.0: ; %entry
4970; GFX10-WGP-NEXT:    s_clause 0x1
4971; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4972; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4973; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4974; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4975; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4976; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4977; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4978; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4979; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
4980; GFX10-WGP-NEXT:    s_endpgm
4981;
4982; GFX10-CU-LABEL: flat_workgroup_one_as_release_atomicrmw:
4983; GFX10-CU:       ; %bb.0: ; %entry
4984; GFX10-CU-NEXT:    s_clause 0x1
4985; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4986; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4987; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4988; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4989; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4990; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4991; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
4992; GFX10-CU-NEXT:    s_endpgm
4993;
4994; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_atomicrmw:
4995; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4996; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4997; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
4998; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4999; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5000; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5001; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5002; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
5003; SKIP-CACHE-INV-NEXT:    s_endpgm
5004;
5005; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw:
5006; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5007; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5008; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5009; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5010; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5011; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5012; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
5013; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5014;
5015; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw:
5016; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5017; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5018; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5019; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5020; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5021; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5022; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5023; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
5024; GFX90A-TGSPLIT-NEXT:    s_endpgm
5025    i32* %out, i32 %in) {
5026entry:
5027  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") release
5028  ret void
5029}
5030
5031define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_atomicrmw(
5032; GFX7-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
5033; GFX7:       ; %bb.0: ; %entry
5034; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5035; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5036; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5037; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5038; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5039; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5040; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
5041; GFX7-NEXT:    s_endpgm
5042;
5043; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
5044; GFX10-WGP:       ; %bb.0: ; %entry
5045; GFX10-WGP-NEXT:    s_clause 0x1
5046; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5047; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5048; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5049; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5050; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5051; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5052; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5053; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5054; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
5055; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5056; GFX10-WGP-NEXT:    buffer_gl0_inv
5057; GFX10-WGP-NEXT:    s_endpgm
5058;
5059; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
5060; GFX10-CU:       ; %bb.0: ; %entry
5061; GFX10-CU-NEXT:    s_clause 0x1
5062; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5063; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5064; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5065; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5066; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5067; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5068; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
5069; GFX10-CU-NEXT:    s_endpgm
5070;
5071; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
5072; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5073; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5074; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
5075; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5076; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5077; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5078; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5079; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
5080; SKIP-CACHE-INV-NEXT:    s_endpgm
5081;
5082; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
5083; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5084; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5085; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5086; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5087; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5088; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5089; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
5090; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5091;
5092; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
5093; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5094; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5095; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5096; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5097; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5098; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5099; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5100; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
5101; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5102; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5103; GFX90A-TGSPLIT-NEXT:    s_endpgm
5104    i32* %out, i32 %in) {
5105entry:
5106  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acq_rel
5107  ret void
5108}
5109
5110define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_atomicrmw(
5111; GFX7-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
5112; GFX7:       ; %bb.0: ; %entry
5113; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5114; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5115; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5116; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5117; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5118; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5119; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
5120; GFX7-NEXT:    s_endpgm
5121;
5122; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
5123; GFX10-WGP:       ; %bb.0: ; %entry
5124; GFX10-WGP-NEXT:    s_clause 0x1
5125; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5126; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5127; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5128; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5129; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5130; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5131; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5132; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5133; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
5134; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5135; GFX10-WGP-NEXT:    buffer_gl0_inv
5136; GFX10-WGP-NEXT:    s_endpgm
5137;
5138; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
5139; GFX10-CU:       ; %bb.0: ; %entry
5140; GFX10-CU-NEXT:    s_clause 0x1
5141; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5142; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5143; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5144; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5145; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5146; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5147; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
5148; GFX10-CU-NEXT:    s_endpgm
5149;
5150; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
5151; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5152; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5153; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
5154; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5155; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5156; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5157; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5158; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
5159; SKIP-CACHE-INV-NEXT:    s_endpgm
5160;
5161; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
5162; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5163; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5164; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5165; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5166; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5167; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5168; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
5169; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5170;
5171; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
5172; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5173; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5174; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5175; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5176; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5177; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5178; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5179; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
5180; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5181; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5182; GFX90A-TGSPLIT-NEXT:    s_endpgm
5183    i32* %out, i32 %in) {
5184entry:
5185  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") seq_cst
5186  ret void
5187}
5188
5189define amdgpu_kernel void @flat_workgroup_one_as_acquire_ret_atomicrmw(
5190; GFX7-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
5191; GFX7:       ; %bb.0: ; %entry
5192; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5193; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5194; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5195; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5196; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5197; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5198; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5199; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5200; GFX7-NEXT:    flat_store_dword v[0:1], v2
5201; GFX7-NEXT:    s_endpgm
5202;
5203; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
5204; GFX10-WGP:       ; %bb.0: ; %entry
5205; GFX10-WGP-NEXT:    s_clause 0x1
5206; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5207; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5208; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5209; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5210; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5211; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5212; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5213; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5214; GFX10-WGP-NEXT:    buffer_gl0_inv
5215; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5216; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5217; GFX10-WGP-NEXT:    s_endpgm
5218;
5219; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
5220; GFX10-CU:       ; %bb.0: ; %entry
5221; GFX10-CU-NEXT:    s_clause 0x1
5222; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5223; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5224; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5225; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5226; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5227; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5228; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5229; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5230; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5231; GFX10-CU-NEXT:    s_endpgm
5232;
5233; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
5234; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5235; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5236; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
5237; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5238; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5239; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5240; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5241; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5242; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5243; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5244; SKIP-CACHE-INV-NEXT:    s_endpgm
5245;
5246; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
5247; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5248; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5249; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5250; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5251; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5252; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5253; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5254; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5255; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5256; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5257;
5258; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
5259; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5260; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5261; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5262; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5263; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5264; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5265; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5266; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5267; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5268; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5269; GFX90A-TGSPLIT-NEXT:    s_endpgm
5270    i32* %out, i32 %in) {
5271entry:
5272  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acquire
5273  store i32 %val, i32* %out, align 4
5274  ret void
5275}
5276
5277define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_ret_atomicrmw(
5278; GFX7-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
5279; GFX7:       ; %bb.0: ; %entry
5280; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5281; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5282; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5283; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5284; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5285; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5286; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5287; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5288; GFX7-NEXT:    flat_store_dword v[0:1], v2
5289; GFX7-NEXT:    s_endpgm
5290;
5291; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
5292; GFX10-WGP:       ; %bb.0: ; %entry
5293; GFX10-WGP-NEXT:    s_clause 0x1
5294; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5295; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5296; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5297; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5298; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5299; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5300; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5301; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5302; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5303; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5304; GFX10-WGP-NEXT:    buffer_gl0_inv
5305; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5306; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5307; GFX10-WGP-NEXT:    s_endpgm
5308;
5309; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
5310; GFX10-CU:       ; %bb.0: ; %entry
5311; GFX10-CU-NEXT:    s_clause 0x1
5312; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5313; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5314; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5315; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5316; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5317; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5318; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5319; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5320; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5321; GFX10-CU-NEXT:    s_endpgm
5322;
5323; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
5324; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5325; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5326; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
5327; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5328; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5329; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5330; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5331; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5332; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5333; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5334; SKIP-CACHE-INV-NEXT:    s_endpgm
5335;
5336; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
5337; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5338; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5339; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5340; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5341; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5342; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5343; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5344; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5345; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5346; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5347;
5348; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
5349; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5350; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5351; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5352; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5353; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5354; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5355; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5356; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5357; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5358; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5359; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5360; GFX90A-TGSPLIT-NEXT:    s_endpgm
5361    i32* %out, i32 %in) {
5362entry:
5363  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acq_rel
5364  store i32 %val, i32* %out, align 4
5365  ret void
5366}
5367
5368define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_ret_atomicrmw(
5369; GFX7-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
5370; GFX7:       ; %bb.0: ; %entry
5371; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5372; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5373; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5374; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5375; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5376; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5377; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5378; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5379; GFX7-NEXT:    flat_store_dword v[0:1], v2
5380; GFX7-NEXT:    s_endpgm
5381;
5382; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
5383; GFX10-WGP:       ; %bb.0: ; %entry
5384; GFX10-WGP-NEXT:    s_clause 0x1
5385; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5386; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5387; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5388; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5389; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5390; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5391; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5392; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5393; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5394; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5395; GFX10-WGP-NEXT:    buffer_gl0_inv
5396; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5397; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5398; GFX10-WGP-NEXT:    s_endpgm
5399;
5400; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
5401; GFX10-CU:       ; %bb.0: ; %entry
5402; GFX10-CU-NEXT:    s_clause 0x1
5403; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5404; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5405; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5406; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5407; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5408; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5409; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5410; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5411; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5412; GFX10-CU-NEXT:    s_endpgm
5413;
5414; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
5415; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5416; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5417; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
5418; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5419; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5420; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5421; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5422; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5423; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5424; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5425; SKIP-CACHE-INV-NEXT:    s_endpgm
5426;
5427; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
5428; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5429; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5430; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5431; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5432; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5433; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5434; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5435; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5436; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5437; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5438;
5439; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
5440; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5441; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5442; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5443; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5444; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5445; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5446; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5447; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5448; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5449; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5450; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5451; GFX90A-TGSPLIT-NEXT:    s_endpgm
5452    i32* %out, i32 %in) {
5453entry:
5454  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") seq_cst
5455  store i32 %val, i32* %out, align 4
5456  ret void
5457}
5458
5459define amdgpu_kernel void @flat_workgroup_one_as_monotonic_monotonic_cmpxchg(
5460; GFX7-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
5461; GFX7:       ; %bb.0: ; %entry
5462; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5463; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5464; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5465; GFX7-NEXT:    s_add_u32 s0, s0, 16
5466; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5467; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5468; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5469; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5470; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5471; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5472; GFX7-NEXT:    s_endpgm
5473;
5474; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
5475; GFX10-WGP:       ; %bb.0: ; %entry
5476; GFX10-WGP-NEXT:    s_clause 0x1
5477; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5478; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5479; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5480; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5481; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5482; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5483; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5484; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5485; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5486; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5487; GFX10-WGP-NEXT:    s_endpgm
5488;
5489; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
5490; GFX10-CU:       ; %bb.0: ; %entry
5491; GFX10-CU-NEXT:    s_clause 0x1
5492; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5493; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5494; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5495; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5496; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5497; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5498; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5499; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5500; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5501; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5502; GFX10-CU-NEXT:    s_endpgm
5503;
5504; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
5505; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5506; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5507; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5508; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5509; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5510; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5511; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5512; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5513; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5514; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5515; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5516; SKIP-CACHE-INV-NEXT:    s_endpgm
5517;
5518; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
5519; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5520; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5521; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5522; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5523; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5524; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5525; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5526; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5527;
5528; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
5529; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5530; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5531; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5532; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5533; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5534; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5535; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5536; GFX90A-TGSPLIT-NEXT:    s_endpgm
5537    i32* %out, i32 %in, i32 %old) {
5538entry:
5539  %gep = getelementptr i32, i32* %out, i32 4
5540  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic
5541  ret void
5542}
5543
5544define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_cmpxchg(
5545; GFX7-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
5546; GFX7:       ; %bb.0: ; %entry
5547; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5548; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5549; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5550; GFX7-NEXT:    s_add_u32 s0, s0, 16
5551; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5552; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5553; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5554; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5555; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5556; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5557; GFX7-NEXT:    s_endpgm
5558;
5559; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
5560; GFX10-WGP:       ; %bb.0: ; %entry
5561; GFX10-WGP-NEXT:    s_clause 0x1
5562; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5563; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5564; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5565; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5566; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5567; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5568; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5569; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5570; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5571; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5572; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5573; GFX10-WGP-NEXT:    buffer_gl0_inv
5574; GFX10-WGP-NEXT:    s_endpgm
5575;
5576; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
5577; GFX10-CU:       ; %bb.0: ; %entry
5578; GFX10-CU-NEXT:    s_clause 0x1
5579; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5580; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5581; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5582; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5583; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5584; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5585; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5586; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5587; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5588; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5589; GFX10-CU-NEXT:    s_endpgm
5590;
5591; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
5592; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5593; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5594; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5595; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5596; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5597; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5598; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5599; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5600; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5601; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5602; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5603; SKIP-CACHE-INV-NEXT:    s_endpgm
5604;
5605; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
5606; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5607; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5608; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5609; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5610; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5611; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5612; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5613; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5614;
5615; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
5616; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5617; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5618; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5619; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5620; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5621; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5622; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5623; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5624; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5625; GFX90A-TGSPLIT-NEXT:    s_endpgm
5626    i32* %out, i32 %in, i32 %old) {
5627entry:
5628  %gep = getelementptr i32, i32* %out, i32 4
5629  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
5630  ret void
5631}
5632
5633define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_cmpxchg(
5634; GFX7-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
5635; GFX7:       ; %bb.0: ; %entry
5636; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5637; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5638; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5639; GFX7-NEXT:    s_add_u32 s0, s0, 16
5640; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5641; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5642; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5643; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5644; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5645; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5646; GFX7-NEXT:    s_endpgm
5647;
5648; GFX10-WGP-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
5649; GFX10-WGP:       ; %bb.0: ; %entry
5650; GFX10-WGP-NEXT:    s_clause 0x1
5651; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5652; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5653; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5654; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5655; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5656; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5657; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5658; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5659; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5660; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5661; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5662; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5663; GFX10-WGP-NEXT:    s_endpgm
5664;
5665; GFX10-CU-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
5666; GFX10-CU:       ; %bb.0: ; %entry
5667; GFX10-CU-NEXT:    s_clause 0x1
5668; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5669; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5670; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5671; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5672; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5673; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5674; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5675; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5676; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5677; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5678; GFX10-CU-NEXT:    s_endpgm
5679;
5680; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
5681; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5682; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5683; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5684; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5685; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5686; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5687; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5688; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5689; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5690; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5691; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5692; SKIP-CACHE-INV-NEXT:    s_endpgm
5693;
5694; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
5695; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5696; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5697; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5698; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5699; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5700; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5701; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5702; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5703;
5704; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
5705; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5706; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5707; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5708; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5709; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5710; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5711; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5712; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5713; GFX90A-TGSPLIT-NEXT:    s_endpgm
5714    i32* %out, i32 %in, i32 %old) {
5715entry:
5716  %gep = getelementptr i32, i32* %out, i32 4
5717  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic
5718  ret void
5719}
5720
5721define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_cmpxchg(
5722; GFX7-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
5723; GFX7:       ; %bb.0: ; %entry
5724; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5725; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5726; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5727; GFX7-NEXT:    s_add_u32 s0, s0, 16
5728; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5729; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5730; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5731; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5732; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5733; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5734; GFX7-NEXT:    s_endpgm
5735;
5736; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
5737; GFX10-WGP:       ; %bb.0: ; %entry
5738; GFX10-WGP-NEXT:    s_clause 0x1
5739; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5740; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5741; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5742; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5743; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5744; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5745; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5746; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5747; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5748; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5749; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5750; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5751; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5752; GFX10-WGP-NEXT:    buffer_gl0_inv
5753; GFX10-WGP-NEXT:    s_endpgm
5754;
5755; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
5756; GFX10-CU:       ; %bb.0: ; %entry
5757; GFX10-CU-NEXT:    s_clause 0x1
5758; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5759; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5760; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5761; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5762; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5763; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5764; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5765; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5766; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5767; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5768; GFX10-CU-NEXT:    s_endpgm
5769;
5770; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
5771; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5772; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5773; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5774; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5775; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5776; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5777; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5778; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5779; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5780; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5781; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5782; SKIP-CACHE-INV-NEXT:    s_endpgm
5783;
5784; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
5785; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5786; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5787; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5788; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5789; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5790; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5791; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5792; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5793;
5794; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
5795; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5796; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5797; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5798; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5799; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5800; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5801; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5802; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5803; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5804; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5805; GFX90A-TGSPLIT-NEXT:    s_endpgm
5806    i32* %out, i32 %in, i32 %old) {
5807entry:
5808  %gep = getelementptr i32, i32* %out, i32 4
5809  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
5810  ret void
5811}
5812
5813define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_cmpxchg(
5814; GFX7-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
5815; GFX7:       ; %bb.0: ; %entry
5816; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5817; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5818; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5819; GFX7-NEXT:    s_add_u32 s0, s0, 16
5820; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5821; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5822; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5823; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5824; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5825; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5826; GFX7-NEXT:    s_endpgm
5827;
5828; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
5829; GFX10-WGP:       ; %bb.0: ; %entry
5830; GFX10-WGP-NEXT:    s_clause 0x1
5831; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5832; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5833; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5834; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5835; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5836; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5837; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5838; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5839; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5840; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5841; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5842; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5843; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5844; GFX10-WGP-NEXT:    buffer_gl0_inv
5845; GFX10-WGP-NEXT:    s_endpgm
5846;
5847; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
5848; GFX10-CU:       ; %bb.0: ; %entry
5849; GFX10-CU-NEXT:    s_clause 0x1
5850; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5851; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5852; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5853; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5854; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5855; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5856; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5857; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5858; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5859; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5860; GFX10-CU-NEXT:    s_endpgm
5861;
5862; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
5863; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5864; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5865; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5866; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5867; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5868; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5869; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5870; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5871; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5872; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5873; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5874; SKIP-CACHE-INV-NEXT:    s_endpgm
5875;
5876; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
5877; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5878; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5879; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5880; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5881; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5882; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5883; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5884; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5885;
5886; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
5887; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5888; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5889; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5890; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5891; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5892; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5893; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5894; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5895; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5896; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5897; GFX90A-TGSPLIT-NEXT:    s_endpgm
5898    i32* %out, i32 %in, i32 %old) {
5899entry:
5900  %gep = getelementptr i32, i32* %out, i32 4
5901  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
5902  ret void
5903}
5904
5905define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_cmpxchg(
5906; GFX7-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
5907; GFX7:       ; %bb.0: ; %entry
5908; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5909; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5910; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5911; GFX7-NEXT:    s_add_u32 s0, s0, 16
5912; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5913; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5914; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5915; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5916; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5917; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5918; GFX7-NEXT:    s_endpgm
5919;
5920; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
5921; GFX10-WGP:       ; %bb.0: ; %entry
5922; GFX10-WGP-NEXT:    s_clause 0x1
5923; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5924; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5925; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5926; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5927; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5928; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5929; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5930; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5931; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5932; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5933; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5934; GFX10-WGP-NEXT:    buffer_gl0_inv
5935; GFX10-WGP-NEXT:    s_endpgm
5936;
5937; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
5938; GFX10-CU:       ; %bb.0: ; %entry
5939; GFX10-CU-NEXT:    s_clause 0x1
5940; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5941; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5942; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5943; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5944; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5945; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5946; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5947; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5948; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5949; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5950; GFX10-CU-NEXT:    s_endpgm
5951;
5952; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
5953; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5954; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5955; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5956; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5957; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5958; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5959; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5960; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5961; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5962; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5963; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5964; SKIP-CACHE-INV-NEXT:    s_endpgm
5965;
5966; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
5967; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5968; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5969; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5970; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5971; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5972; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5973; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5974; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5975;
5976; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
5977; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5978; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5979; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5980; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5981; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5982; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5983; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5984; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5985; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5986; GFX90A-TGSPLIT-NEXT:    s_endpgm
5987    i32* %out, i32 %in, i32 %old) {
5988entry:
5989  %gep = getelementptr i32, i32* %out, i32 4
5990  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic acquire
5991  ret void
5992}
5993
5994define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_cmpxchg(
5995; GFX7-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
5996; GFX7:       ; %bb.0: ; %entry
5997; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5998; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5999; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6000; GFX7-NEXT:    s_add_u32 s0, s0, 16
6001; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6002; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6003; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6004; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6005; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6006; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6007; GFX7-NEXT:    s_endpgm
6008;
6009; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
6010; GFX10-WGP:       ; %bb.0: ; %entry
6011; GFX10-WGP-NEXT:    s_clause 0x1
6012; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6013; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6014; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6015; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6016; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6017; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6018; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6019; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6020; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6021; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6022; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6023; GFX10-WGP-NEXT:    buffer_gl0_inv
6024; GFX10-WGP-NEXT:    s_endpgm
6025;
6026; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
6027; GFX10-CU:       ; %bb.0: ; %entry
6028; GFX10-CU-NEXT:    s_clause 0x1
6029; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6030; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6031; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6032; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6033; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6034; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6035; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6036; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6037; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6038; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6039; GFX10-CU-NEXT:    s_endpgm
6040;
6041; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
6042; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6043; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6044; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6045; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6046; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6047; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6048; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6049; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6050; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6051; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6052; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6053; SKIP-CACHE-INV-NEXT:    s_endpgm
6054;
6055; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
6056; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6057; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6058; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6059; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6060; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6061; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6062; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6063; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6064;
6065; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
6066; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6067; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6068; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6069; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6070; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6071; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6072; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6073; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6074; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6075; GFX90A-TGSPLIT-NEXT:    s_endpgm
6076    i32* %out, i32 %in, i32 %old) {
6077entry:
6078  %gep = getelementptr i32, i32* %out, i32 4
6079  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
6080  ret void
6081}
6082
6083define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_cmpxchg(
6084; GFX7-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
6085; GFX7:       ; %bb.0: ; %entry
6086; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6087; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6088; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6089; GFX7-NEXT:    s_add_u32 s0, s0, 16
6090; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6091; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6092; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6093; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6094; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6095; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6096; GFX7-NEXT:    s_endpgm
6097;
6098; GFX10-WGP-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
6099; GFX10-WGP:       ; %bb.0: ; %entry
6100; GFX10-WGP-NEXT:    s_clause 0x1
6101; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6102; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6103; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6104; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6105; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6106; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6107; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6108; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6109; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6110; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6111; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6112; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6113; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6114; GFX10-WGP-NEXT:    buffer_gl0_inv
6115; GFX10-WGP-NEXT:    s_endpgm
6116;
6117; GFX10-CU-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
6118; GFX10-CU:       ; %bb.0: ; %entry
6119; GFX10-CU-NEXT:    s_clause 0x1
6120; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6121; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6122; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6123; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6124; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6125; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6126; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6127; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6128; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6129; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6130; GFX10-CU-NEXT:    s_endpgm
6131;
6132; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
6133; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6134; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6135; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6136; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6137; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6138; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6139; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6140; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6141; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6142; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6143; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6144; SKIP-CACHE-INV-NEXT:    s_endpgm
6145;
6146; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
6147; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6148; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6149; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6150; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6151; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6152; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6153; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6154; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6155;
6156; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
6157; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6158; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6159; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6160; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6161; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6162; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6163; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6164; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6165; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6166; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6167; GFX90A-TGSPLIT-NEXT:    s_endpgm
6168    i32* %out, i32 %in, i32 %old) {
6169entry:
6170  %gep = getelementptr i32, i32* %out, i32 4
6171  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
6172  ret void
6173}
6174
6175define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_cmpxchg(
6176; GFX7-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
6177; GFX7:       ; %bb.0: ; %entry
6178; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6179; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6180; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6181; GFX7-NEXT:    s_add_u32 s0, s0, 16
6182; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6183; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6184; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6185; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6186; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6187; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6188; GFX7-NEXT:    s_endpgm
6189;
6190; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
6191; GFX10-WGP:       ; %bb.0: ; %entry
6192; GFX10-WGP-NEXT:    s_clause 0x1
6193; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6194; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6195; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6196; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6197; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6198; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6199; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6200; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6201; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6202; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6203; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6204; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6205; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6206; GFX10-WGP-NEXT:    buffer_gl0_inv
6207; GFX10-WGP-NEXT:    s_endpgm
6208;
6209; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
6210; GFX10-CU:       ; %bb.0: ; %entry
6211; GFX10-CU-NEXT:    s_clause 0x1
6212; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6213; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6214; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6215; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6216; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6217; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6218; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6219; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6220; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6221; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6222; GFX10-CU-NEXT:    s_endpgm
6223;
6224; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
6225; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6226; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6227; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6228; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6229; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6230; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6231; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6232; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6233; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6234; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6235; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6236; SKIP-CACHE-INV-NEXT:    s_endpgm
6237;
6238; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
6239; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6240; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6241; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6242; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6243; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6244; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6245; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6246; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6247;
6248; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
6249; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6250; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6251; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6252; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6253; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6254; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6255; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6256; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6257; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6258; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6259; GFX90A-TGSPLIT-NEXT:    s_endpgm
6260    i32* %out, i32 %in, i32 %old) {
6261entry:
6262  %gep = getelementptr i32, i32* %out, i32 4
6263  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
6264  ret void
6265}
6266
6267define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_cmpxchg(
6268; GFX7-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
6269; GFX7:       ; %bb.0: ; %entry
6270; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6271; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6272; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6273; GFX7-NEXT:    s_add_u32 s0, s0, 16
6274; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6275; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6276; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6277; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6278; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6279; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6280; GFX7-NEXT:    s_endpgm
6281;
6282; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
6283; GFX10-WGP:       ; %bb.0: ; %entry
6284; GFX10-WGP-NEXT:    s_clause 0x1
6285; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6286; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6287; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6288; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6289; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6290; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6291; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6292; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6293; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6294; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6295; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6296; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6297; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6298; GFX10-WGP-NEXT:    buffer_gl0_inv
6299; GFX10-WGP-NEXT:    s_endpgm
6300;
6301; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
6302; GFX10-CU:       ; %bb.0: ; %entry
6303; GFX10-CU-NEXT:    s_clause 0x1
6304; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6305; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6306; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6307; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6308; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6309; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6310; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6311; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6312; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6313; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6314; GFX10-CU-NEXT:    s_endpgm
6315;
6316; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
6317; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6318; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6319; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6320; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6321; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6322; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6323; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6324; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6325; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6326; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6327; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6328; SKIP-CACHE-INV-NEXT:    s_endpgm
6329;
6330; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
6331; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6332; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6333; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6334; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6335; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6336; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6337; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6338; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6339;
6340; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
6341; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6342; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6343; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6344; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6345; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6346; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6347; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6348; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6349; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6350; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6351; GFX90A-TGSPLIT-NEXT:    s_endpgm
6352    i32* %out, i32 %in, i32 %old) {
6353entry:
6354  %gep = getelementptr i32, i32* %out, i32 4
6355  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
6356  ret void
6357}
6358
6359define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_cmpxchg(
6360; GFX7-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
6361; GFX7:       ; %bb.0: ; %entry
6362; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6363; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6364; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6365; GFX7-NEXT:    s_add_u32 s0, s0, 16
6366; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6367; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6368; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6369; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6370; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6371; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6372; GFX7-NEXT:    s_endpgm
6373;
6374; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
6375; GFX10-WGP:       ; %bb.0: ; %entry
6376; GFX10-WGP-NEXT:    s_clause 0x1
6377; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6378; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6379; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6380; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6381; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6382; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6383; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6384; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6385; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6386; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6387; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6388; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6389; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6390; GFX10-WGP-NEXT:    buffer_gl0_inv
6391; GFX10-WGP-NEXT:    s_endpgm
6392;
6393; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
6394; GFX10-CU:       ; %bb.0: ; %entry
6395; GFX10-CU-NEXT:    s_clause 0x1
6396; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6397; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6398; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6399; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6400; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6401; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6402; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6403; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6404; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6405; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6406; GFX10-CU-NEXT:    s_endpgm
6407;
6408; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
6409; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6410; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6411; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6412; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6413; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6414; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6415; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6416; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6417; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6418; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6419; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6420; SKIP-CACHE-INV-NEXT:    s_endpgm
6421;
6422; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
6423; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6424; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6425; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6426; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6427; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6428; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6429; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6430; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6431;
6432; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
6433; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6434; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6435; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6436; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6437; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6438; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6439; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6440; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6441; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6442; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6443; GFX90A-TGSPLIT-NEXT:    s_endpgm
6444    i32* %out, i32 %in, i32 %old) {
6445entry:
6446  %gep = getelementptr i32, i32* %out, i32 4
6447  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic seq_cst
6448  ret void
6449}
6450
6451define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_cmpxchg(
6452; GFX7-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
6453; GFX7:       ; %bb.0: ; %entry
6454; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6455; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6456; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6457; GFX7-NEXT:    s_add_u32 s0, s0, 16
6458; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6459; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6460; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6461; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6462; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6463; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6464; GFX7-NEXT:    s_endpgm
6465;
6466; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
6467; GFX10-WGP:       ; %bb.0: ; %entry
6468; GFX10-WGP-NEXT:    s_clause 0x1
6469; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6470; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6471; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6472; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6473; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6474; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6475; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6476; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6477; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6478; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6479; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6480; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6481; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6482; GFX10-WGP-NEXT:    buffer_gl0_inv
6483; GFX10-WGP-NEXT:    s_endpgm
6484;
6485; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
6486; GFX10-CU:       ; %bb.0: ; %entry
6487; GFX10-CU-NEXT:    s_clause 0x1
6488; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6489; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6490; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6491; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6492; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6493; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6494; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6495; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6496; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6497; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6498; GFX10-CU-NEXT:    s_endpgm
6499;
6500; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
6501; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6502; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6503; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6504; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6505; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6506; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6507; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6508; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6509; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6510; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6511; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6512; SKIP-CACHE-INV-NEXT:    s_endpgm
6513;
6514; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
6515; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6516; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6517; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6518; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6519; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6520; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6521; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6522; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6523;
6524; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
6525; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6526; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6527; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6528; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6529; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6530; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6531; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6532; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6533; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6534; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6535; GFX90A-TGSPLIT-NEXT:    s_endpgm
6536    i32* %out, i32 %in, i32 %old) {
6537entry:
6538  %gep = getelementptr i32, i32* %out, i32 4
6539  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire seq_cst
6540  ret void
6541}
6542
6543define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_cmpxchg(
6544; GFX7-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
6545; GFX7:       ; %bb.0: ; %entry
6546; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6547; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6548; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6549; GFX7-NEXT:    s_add_u32 s0, s0, 16
6550; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6551; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6552; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6553; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6554; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6555; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6556; GFX7-NEXT:    s_endpgm
6557;
6558; GFX10-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
6559; GFX10-WGP:       ; %bb.0: ; %entry
6560; GFX10-WGP-NEXT:    s_clause 0x1
6561; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6562; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6563; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6564; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6565; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6566; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6567; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6568; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6569; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6570; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6571; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6572; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6573; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6574; GFX10-WGP-NEXT:    buffer_gl0_inv
6575; GFX10-WGP-NEXT:    s_endpgm
6576;
6577; GFX10-CU-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
6578; GFX10-CU:       ; %bb.0: ; %entry
6579; GFX10-CU-NEXT:    s_clause 0x1
6580; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6581; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6582; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6583; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6584; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6585; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6586; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6587; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6588; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6589; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6590; GFX10-CU-NEXT:    s_endpgm
6591;
6592; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
6593; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6594; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6595; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6596; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6597; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6598; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6599; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6600; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6601; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6602; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6603; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6604; SKIP-CACHE-INV-NEXT:    s_endpgm
6605;
6606; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
6607; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6608; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6609; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6610; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6611; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6612; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6613; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6614; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6615;
6616; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
6617; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6618; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6619; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6620; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6621; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6622; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6623; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6624; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6625; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6626; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6627; GFX90A-TGSPLIT-NEXT:    s_endpgm
6628    i32* %out, i32 %in, i32 %old) {
6629entry:
6630  %gep = getelementptr i32, i32* %out, i32 4
6631  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release seq_cst
6632  ret void
6633}
6634
6635define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
6636; GFX7-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
6637; GFX7:       ; %bb.0: ; %entry
6638; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6639; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6640; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6641; GFX7-NEXT:    s_add_u32 s0, s0, 16
6642; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6643; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6644; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6645; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6646; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6647; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6648; GFX7-NEXT:    s_endpgm
6649;
6650; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
6651; GFX10-WGP:       ; %bb.0: ; %entry
6652; GFX10-WGP-NEXT:    s_clause 0x1
6653; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6654; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6655; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6656; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6657; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6658; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6659; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6660; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6661; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6662; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6663; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6664; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6665; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6666; GFX10-WGP-NEXT:    buffer_gl0_inv
6667; GFX10-WGP-NEXT:    s_endpgm
6668;
6669; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
6670; GFX10-CU:       ; %bb.0: ; %entry
6671; GFX10-CU-NEXT:    s_clause 0x1
6672; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6673; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6674; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6675; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6676; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6677; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6678; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6679; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6680; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6681; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6682; GFX10-CU-NEXT:    s_endpgm
6683;
6684; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
6685; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6686; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6687; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6688; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6689; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6690; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6691; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6692; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6693; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6694; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6695; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6696; SKIP-CACHE-INV-NEXT:    s_endpgm
6697;
6698; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
6699; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6700; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6701; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6702; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6703; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6704; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6705; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6706; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6707;
6708; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
6709; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6710; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6711; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6712; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6713; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6714; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6715; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6716; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6717; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6718; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6719; GFX90A-TGSPLIT-NEXT:    s_endpgm
6720    i32* %out, i32 %in, i32 %old) {
6721entry:
6722  %gep = getelementptr i32, i32* %out, i32 4
6723  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel seq_cst
6724  ret void
6725}
6726
6727define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
6728; GFX7-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
6729; GFX7:       ; %bb.0: ; %entry
6730; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6731; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6732; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6733; GFX7-NEXT:    s_add_u32 s0, s0, 16
6734; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6735; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6736; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6737; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6738; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6739; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6740; GFX7-NEXT:    s_endpgm
6741;
6742; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
6743; GFX10-WGP:       ; %bb.0: ; %entry
6744; GFX10-WGP-NEXT:    s_clause 0x1
6745; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6746; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6747; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6748; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6749; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6750; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6751; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6752; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6753; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6754; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6755; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6756; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6757; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6758; GFX10-WGP-NEXT:    buffer_gl0_inv
6759; GFX10-WGP-NEXT:    s_endpgm
6760;
6761; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
6762; GFX10-CU:       ; %bb.0: ; %entry
6763; GFX10-CU-NEXT:    s_clause 0x1
6764; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6765; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6766; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6767; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6768; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6769; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6770; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6771; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6772; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6773; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6774; GFX10-CU-NEXT:    s_endpgm
6775;
6776; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
6777; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6778; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6779; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6780; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6781; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6782; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6783; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6784; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6785; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6786; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6787; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6788; SKIP-CACHE-INV-NEXT:    s_endpgm
6789;
6790; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
6791; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6792; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6793; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6794; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6795; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6796; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6797; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6798; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6799;
6800; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
6801; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6802; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6803; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6804; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6805; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6806; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6807; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6808; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6809; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6810; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6811; GFX90A-TGSPLIT-NEXT:    s_endpgm
6812    i32* %out, i32 %in, i32 %old) {
6813entry:
6814  %gep = getelementptr i32, i32* %out, i32 4
6815  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
6816  ret void
6817}
6818
6819define amdgpu_kernel void @flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg(
6820; GFX7-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
6821; GFX7:       ; %bb.0: ; %entry
6822; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6823; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6824; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6825; GFX7-NEXT:    s_add_u32 s4, s0, 16
6826; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6827; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6828; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6829; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6830; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6831; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6832; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6833; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6834; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6835; GFX7-NEXT:    flat_store_dword v[0:1], v2
6836; GFX7-NEXT:    s_endpgm
6837;
6838; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
6839; GFX10-WGP:       ; %bb.0: ; %entry
6840; GFX10-WGP-NEXT:    s_clause 0x1
6841; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6842; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6843; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6844; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6845; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6846; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6847; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6848; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6849; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6850; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6851; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6852; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6853; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6854; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6855; GFX10-WGP-NEXT:    s_endpgm
6856;
6857; GFX10-CU-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
6858; GFX10-CU:       ; %bb.0: ; %entry
6859; GFX10-CU-NEXT:    s_clause 0x1
6860; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6861; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6862; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6863; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6864; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6865; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6866; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6867; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6868; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6869; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6870; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6871; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6872; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6873; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6874; GFX10-CU-NEXT:    s_endpgm
6875;
6876; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
6877; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6878; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6879; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6880; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6881; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6882; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6883; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6884; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6885; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6886; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6887; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6888; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6889; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6890; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6891; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6892; SKIP-CACHE-INV-NEXT:    s_endpgm
6893;
6894; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
6895; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6896; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6897; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6898; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6899; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6900; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6901; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6902; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6903; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6904; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6905;
6906; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
6907; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6908; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6909; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6910; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6911; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6912; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6913; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6914; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6915; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6916; GFX90A-TGSPLIT-NEXT:    s_endpgm
6917    i32* %out, i32 %in, i32 %old) {
6918entry:
6919  %gep = getelementptr i32, i32* %out, i32 4
6920  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic
6921  %val0 = extractvalue { i32, i1 } %val, 0
6922  store i32 %val0, i32* %out, align 4
6923  ret void
6924}
6925
6926define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
6927; GFX7-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
6928; GFX7:       ; %bb.0: ; %entry
6929; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6930; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6931; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6932; GFX7-NEXT:    s_add_u32 s4, s0, 16
6933; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6934; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6935; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6936; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6937; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6938; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6939; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6940; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6941; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6942; GFX7-NEXT:    flat_store_dword v[0:1], v2
6943; GFX7-NEXT:    s_endpgm
6944;
6945; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
6946; GFX10-WGP:       ; %bb.0: ; %entry
6947; GFX10-WGP-NEXT:    s_clause 0x1
6948; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6949; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6950; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6951; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6952; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6953; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6954; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6955; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6956; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6957; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6958; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6959; GFX10-WGP-NEXT:    buffer_gl0_inv
6960; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6961; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6962; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6963; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6964; GFX10-WGP-NEXT:    s_endpgm
6965;
6966; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
6967; GFX10-CU:       ; %bb.0: ; %entry
6968; GFX10-CU-NEXT:    s_clause 0x1
6969; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6970; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6971; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6972; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6973; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6974; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6975; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6976; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6977; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6978; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6979; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6980; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6981; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6982; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6983; GFX10-CU-NEXT:    s_endpgm
6984;
6985; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
6986; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6987; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6988; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6989; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6990; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6991; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6992; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6993; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6994; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6995; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6996; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6997; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6998; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6999; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7000; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7001; SKIP-CACHE-INV-NEXT:    s_endpgm
7002;
7003; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
7004; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7005; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7006; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7007; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7008; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7009; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7010; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7011; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7012; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7013; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7014;
7015; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
7016; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7017; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7018; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7019; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7020; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7021; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7022; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7023; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7024; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7025; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7026; GFX90A-TGSPLIT-NEXT:    s_endpgm
7027    i32* %out, i32 %in, i32 %old) {
7028entry:
7029  %gep = getelementptr i32, i32* %out, i32 4
7030  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
7031  %val0 = extractvalue { i32, i1 } %val, 0
7032  store i32 %val0, i32* %out, align 4
7033  ret void
7034}
7035
7036define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_ret_cmpxchg(
7037; GFX7-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
7038; GFX7:       ; %bb.0: ; %entry
7039; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7040; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7041; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7042; GFX7-NEXT:    s_add_u32 s4, s0, 16
7043; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7044; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7045; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7046; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7047; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7048; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7049; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7050; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7051; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7052; GFX7-NEXT:    flat_store_dword v[0:1], v2
7053; GFX7-NEXT:    s_endpgm
7054;
7055; GFX10-WGP-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
7056; GFX10-WGP:       ; %bb.0: ; %entry
7057; GFX10-WGP-NEXT:    s_clause 0x1
7058; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7059; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7060; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7061; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7062; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7063; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7064; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7065; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7066; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7067; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7068; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7069; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7070; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7071; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7072; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7073; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7074; GFX10-WGP-NEXT:    s_endpgm
7075;
7076; GFX10-CU-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
7077; GFX10-CU:       ; %bb.0: ; %entry
7078; GFX10-CU-NEXT:    s_clause 0x1
7079; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7080; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7081; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7082; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7083; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7084; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7085; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7086; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7087; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7088; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7089; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7090; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7091; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7092; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7093; GFX10-CU-NEXT:    s_endpgm
7094;
7095; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
7096; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7097; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7098; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7099; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7100; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7101; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7102; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7103; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7104; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7105; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7106; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7107; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7108; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7109; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7110; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7111; SKIP-CACHE-INV-NEXT:    s_endpgm
7112;
7113; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
7114; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7115; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7116; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7117; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7118; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7119; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7120; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7121; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7122; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7123; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7124;
7125; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
7126; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7127; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7128; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7129; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7130; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7131; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7132; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7133; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7134; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7135; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7136; GFX90A-TGSPLIT-NEXT:    s_endpgm
7137    i32* %out, i32 %in, i32 %old) {
7138entry:
7139  %gep = getelementptr i32, i32* %out, i32 4
7140  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic
7141  %val0 = extractvalue { i32, i1 } %val, 0
7142  store i32 %val0, i32* %out, align 4
7143  ret void
7144}
7145
7146define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
7147; GFX7-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
7148; GFX7:       ; %bb.0: ; %entry
7149; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7150; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7151; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7152; GFX7-NEXT:    s_add_u32 s4, s0, 16
7153; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7154; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7155; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7156; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7157; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7158; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7159; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7160; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7161; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7162; GFX7-NEXT:    flat_store_dword v[0:1], v2
7163; GFX7-NEXT:    s_endpgm
7164;
7165; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
7166; GFX10-WGP:       ; %bb.0: ; %entry
7167; GFX10-WGP-NEXT:    s_clause 0x1
7168; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7169; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7170; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7171; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7172; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7173; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7174; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7175; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7176; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7177; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7178; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7179; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7180; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7181; GFX10-WGP-NEXT:    buffer_gl0_inv
7182; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7183; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7184; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7185; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7186; GFX10-WGP-NEXT:    s_endpgm
7187;
7188; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
7189; GFX10-CU:       ; %bb.0: ; %entry
7190; GFX10-CU-NEXT:    s_clause 0x1
7191; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7192; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7193; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7194; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7195; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7196; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7197; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7198; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7199; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7200; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7201; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7202; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7203; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7204; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7205; GFX10-CU-NEXT:    s_endpgm
7206;
7207; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
7208; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7209; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7210; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7211; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7212; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7213; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7214; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7215; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7216; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7217; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7218; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7219; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7220; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7221; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7222; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7223; SKIP-CACHE-INV-NEXT:    s_endpgm
7224;
7225; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
7226; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7227; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7228; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7229; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7230; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7231; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7232; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7233; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7234; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7235; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7236;
7237; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
7238; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7239; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7240; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7241; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7242; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7243; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7244; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7245; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7246; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7247; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7248; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7249; GFX90A-TGSPLIT-NEXT:    s_endpgm
7250    i32* %out, i32 %in, i32 %old) {
7251entry:
7252  %gep = getelementptr i32, i32* %out, i32 4
7253  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
7254  %val0 = extractvalue { i32, i1 } %val, 0
7255  store i32 %val0, i32* %out, align 4
7256  ret void
7257}
7258
7259define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
7260; GFX7-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
7261; GFX7:       ; %bb.0: ; %entry
7262; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7263; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7264; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7265; GFX7-NEXT:    s_add_u32 s4, s0, 16
7266; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7267; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7268; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7269; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7270; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7271; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7272; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7273; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7274; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7275; GFX7-NEXT:    flat_store_dword v[0:1], v2
7276; GFX7-NEXT:    s_endpgm
7277;
7278; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
7279; GFX10-WGP:       ; %bb.0: ; %entry
7280; GFX10-WGP-NEXT:    s_clause 0x1
7281; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7282; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7283; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7284; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7285; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7286; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7287; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7288; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7289; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7290; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7291; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7292; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7293; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7294; GFX10-WGP-NEXT:    buffer_gl0_inv
7295; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7296; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7297; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7298; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7299; GFX10-WGP-NEXT:    s_endpgm
7300;
7301; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
7302; GFX10-CU:       ; %bb.0: ; %entry
7303; GFX10-CU-NEXT:    s_clause 0x1
7304; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7305; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7306; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7307; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7308; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7309; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7310; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7311; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7312; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7313; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7314; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7315; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7316; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7317; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7318; GFX10-CU-NEXT:    s_endpgm
7319;
7320; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
7321; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7322; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7323; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7324; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7325; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7326; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7327; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7328; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7329; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7330; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7331; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7332; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7333; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7334; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7335; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7336; SKIP-CACHE-INV-NEXT:    s_endpgm
7337;
7338; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
7339; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7340; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7341; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7342; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7343; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7344; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7345; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7346; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7347; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7348; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7349;
7350; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
7351; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7352; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7353; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7354; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7355; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7356; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7357; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7358; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7359; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7360; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7361; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7362; GFX90A-TGSPLIT-NEXT:    s_endpgm
7363    i32* %out, i32 %in, i32 %old) {
7364entry:
7365  %gep = getelementptr i32, i32* %out, i32 4
7366  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
7367  %val0 = extractvalue { i32, i1 } %val, 0
7368  store i32 %val0, i32* %out, align 4
7369  ret void
7370}
7371
7372define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg(
7373; GFX7-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
7374; GFX7:       ; %bb.0: ; %entry
7375; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7376; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7377; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7378; GFX7-NEXT:    s_add_u32 s4, s0, 16
7379; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7380; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7381; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7382; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7383; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7384; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7385; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7386; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7387; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7388; GFX7-NEXT:    flat_store_dword v[0:1], v2
7389; GFX7-NEXT:    s_endpgm
7390;
7391; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
7392; GFX10-WGP:       ; %bb.0: ; %entry
7393; GFX10-WGP-NEXT:    s_clause 0x1
7394; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7395; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7396; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7397; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7398; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7399; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7400; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7401; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7402; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7403; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7404; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7405; GFX10-WGP-NEXT:    buffer_gl0_inv
7406; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7407; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7408; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7409; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7410; GFX10-WGP-NEXT:    s_endpgm
7411;
7412; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
7413; GFX10-CU:       ; %bb.0: ; %entry
7414; GFX10-CU-NEXT:    s_clause 0x1
7415; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7416; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7417; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7418; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7419; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7420; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7421; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7422; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7423; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7424; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7425; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7426; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7427; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7428; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7429; GFX10-CU-NEXT:    s_endpgm
7430;
7431; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
7432; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7433; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7434; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7435; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7436; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7437; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7438; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7439; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7440; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7441; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7442; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7443; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7444; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7445; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7446; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7447; SKIP-CACHE-INV-NEXT:    s_endpgm
7448;
7449; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
7450; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7451; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7452; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7453; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7454; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7455; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7456; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7457; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7458; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7459; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7460;
7461; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
7462; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7463; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7464; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7465; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7466; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7467; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7468; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7469; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7470; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7471; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7472; GFX90A-TGSPLIT-NEXT:    s_endpgm
7473    i32* %out, i32 %in, i32 %old) {
7474entry:
7475  %gep = getelementptr i32, i32* %out, i32 4
7476  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic acquire
7477  %val0 = extractvalue { i32, i1 } %val, 0
7478  store i32 %val0, i32* %out, align 4
7479  ret void
7480}
7481
7482define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_ret_cmpxchg(
7483; GFX7-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
7484; GFX7:       ; %bb.0: ; %entry
7485; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7486; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7487; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7488; GFX7-NEXT:    s_add_u32 s4, s0, 16
7489; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7490; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7491; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7492; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7493; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7494; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7495; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7496; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7497; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7498; GFX7-NEXT:    flat_store_dword v[0:1], v2
7499; GFX7-NEXT:    s_endpgm
7500;
7501; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
7502; GFX10-WGP:       ; %bb.0: ; %entry
7503; GFX10-WGP-NEXT:    s_clause 0x1
7504; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7505; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7506; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7507; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7508; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7509; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7510; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7511; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7512; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7513; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7514; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7515; GFX10-WGP-NEXT:    buffer_gl0_inv
7516; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7517; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7518; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7519; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7520; GFX10-WGP-NEXT:    s_endpgm
7521;
7522; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
7523; GFX10-CU:       ; %bb.0: ; %entry
7524; GFX10-CU-NEXT:    s_clause 0x1
7525; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7526; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7527; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7528; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7529; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7530; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7531; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7532; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7533; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7534; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7535; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7536; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7537; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7538; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7539; GFX10-CU-NEXT:    s_endpgm
7540;
7541; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
7542; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7543; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7544; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7545; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7546; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7547; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7548; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7549; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7550; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7551; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7552; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7553; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7554; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7555; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7556; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7557; SKIP-CACHE-INV-NEXT:    s_endpgm
7558;
7559; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
7560; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7561; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7562; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7563; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7564; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7565; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7566; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7567; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7568; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7569; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7570;
7571; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
7572; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7573; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7574; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7575; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7576; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7577; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7578; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7579; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7580; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7581; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7582; GFX90A-TGSPLIT-NEXT:    s_endpgm
7583    i32* %out, i32 %in, i32 %old) {
7584entry:
7585  %gep = getelementptr i32, i32* %out, i32 4
7586  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
7587  %val0 = extractvalue { i32, i1 } %val, 0
7588  store i32 %val0, i32* %out, align 4
7589  ret void
7590}
7591
7592define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_ret_cmpxchg(
7593; GFX7-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
7594; GFX7:       ; %bb.0: ; %entry
7595; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7596; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7597; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7598; GFX7-NEXT:    s_add_u32 s4, s0, 16
7599; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7600; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7601; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7602; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7603; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7604; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7605; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7606; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7607; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7608; GFX7-NEXT:    flat_store_dword v[0:1], v2
7609; GFX7-NEXT:    s_endpgm
7610;
7611; GFX10-WGP-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
7612; GFX10-WGP:       ; %bb.0: ; %entry
7613; GFX10-WGP-NEXT:    s_clause 0x1
7614; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7615; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7616; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7617; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7618; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7619; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7620; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7621; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7622; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7623; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7624; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7625; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7626; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7627; GFX10-WGP-NEXT:    buffer_gl0_inv
7628; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7629; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7630; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7631; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7632; GFX10-WGP-NEXT:    s_endpgm
7633;
7634; GFX10-CU-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
7635; GFX10-CU:       ; %bb.0: ; %entry
7636; GFX10-CU-NEXT:    s_clause 0x1
7637; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7638; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7639; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7640; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7641; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7642; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7643; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7644; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7645; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7646; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7647; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7648; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7649; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7650; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7651; GFX10-CU-NEXT:    s_endpgm
7652;
7653; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
7654; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7655; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7656; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7657; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7658; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7659; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7660; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7661; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7662; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7663; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7664; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7665; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7666; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7667; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7668; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7669; SKIP-CACHE-INV-NEXT:    s_endpgm
7670;
7671; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
7672; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7673; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7674; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7675; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7676; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7677; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7678; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7679; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7680; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7681; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7682;
7683; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
7684; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7685; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7686; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7687; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7688; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7689; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7690; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7691; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7692; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7693; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7694; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7695; GFX90A-TGSPLIT-NEXT:    s_endpgm
7696    i32* %out, i32 %in, i32 %old) {
7697entry:
7698  %gep = getelementptr i32, i32* %out, i32 4
7699  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
7700  %val0 = extractvalue { i32, i1 } %val, 0
7701  store i32 %val0, i32* %out, align 4
7702  ret void
7703}
7704
7705define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
7706; GFX7-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
7707; GFX7:       ; %bb.0: ; %entry
7708; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7709; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7710; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7711; GFX7-NEXT:    s_add_u32 s4, s0, 16
7712; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7713; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7714; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7715; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7716; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7717; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7718; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7719; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7720; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7721; GFX7-NEXT:    flat_store_dword v[0:1], v2
7722; GFX7-NEXT:    s_endpgm
7723;
7724; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
7725; GFX10-WGP:       ; %bb.0: ; %entry
7726; GFX10-WGP-NEXT:    s_clause 0x1
7727; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7728; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7729; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7730; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7731; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7732; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7733; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7734; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7735; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7736; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7737; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7738; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7739; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7740; GFX10-WGP-NEXT:    buffer_gl0_inv
7741; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7742; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7743; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7744; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7745; GFX10-WGP-NEXT:    s_endpgm
7746;
7747; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
7748; GFX10-CU:       ; %bb.0: ; %entry
7749; GFX10-CU-NEXT:    s_clause 0x1
7750; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7751; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7752; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7753; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7754; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7755; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7756; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7757; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7758; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7759; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7760; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7761; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7762; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7763; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7764; GFX10-CU-NEXT:    s_endpgm
7765;
7766; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
7767; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7768; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7769; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7770; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7771; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7772; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7773; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7774; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7775; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7776; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7777; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7778; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7779; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7780; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7781; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7782; SKIP-CACHE-INV-NEXT:    s_endpgm
7783;
7784; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
7785; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7786; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7787; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7788; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7789; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7790; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7791; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7792; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7793; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7794; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7795;
7796; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
7797; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7798; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7799; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7800; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7801; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7802; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7803; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7804; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7805; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7806; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7807; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7808; GFX90A-TGSPLIT-NEXT:    s_endpgm
7809    i32* %out, i32 %in, i32 %old) {
7810entry:
7811  %gep = getelementptr i32, i32* %out, i32 4
7812  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
7813  %val0 = extractvalue { i32, i1 } %val, 0
7814  store i32 %val0, i32* %out, align 4
7815  ret void
7816}
7817
7818define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
7819; GFX7-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
7820; GFX7:       ; %bb.0: ; %entry
7821; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7822; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7823; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7824; GFX7-NEXT:    s_add_u32 s4, s0, 16
7825; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7826; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7827; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7828; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7829; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7830; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7831; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7832; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7833; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7834; GFX7-NEXT:    flat_store_dword v[0:1], v2
7835; GFX7-NEXT:    s_endpgm
7836;
7837; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
7838; GFX10-WGP:       ; %bb.0: ; %entry
7839; GFX10-WGP-NEXT:    s_clause 0x1
7840; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7841; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7842; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7843; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7844; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7845; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7846; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7847; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7848; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7849; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7850; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7851; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7852; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7853; GFX10-WGP-NEXT:    buffer_gl0_inv
7854; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7855; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7856; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7857; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7858; GFX10-WGP-NEXT:    s_endpgm
7859;
7860; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
7861; GFX10-CU:       ; %bb.0: ; %entry
7862; GFX10-CU-NEXT:    s_clause 0x1
7863; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7864; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7865; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7866; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7867; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7868; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7869; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7870; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7871; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7872; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7873; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7874; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7875; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7876; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7877; GFX10-CU-NEXT:    s_endpgm
7878;
7879; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
7880; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7881; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7882; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7883; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7884; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7885; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7886; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7887; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7888; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7889; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7890; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7891; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7892; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7893; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7894; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7895; SKIP-CACHE-INV-NEXT:    s_endpgm
7896;
7897; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
7898; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7899; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7900; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7901; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7902; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7903; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7904; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7905; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7906; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7907; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7908;
7909; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
7910; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7911; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7912; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7913; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7914; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7915; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7916; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7917; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7918; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7919; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7920; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7921; GFX90A-TGSPLIT-NEXT:    s_endpgm
7922    i32* %out, i32 %in, i32 %old) {
7923entry:
7924  %gep = getelementptr i32, i32* %out, i32 4
7925  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
7926  %val0 = extractvalue { i32, i1 } %val, 0
7927  store i32 %val0, i32* %out, align 4
7928  ret void
7929}
7930
7931define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg(
7932; GFX7-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
7933; GFX7:       ; %bb.0: ; %entry
7934; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7935; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7936; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7937; GFX7-NEXT:    s_add_u32 s4, s0, 16
7938; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7939; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7940; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7941; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7942; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7943; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7944; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7945; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7946; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7947; GFX7-NEXT:    flat_store_dword v[0:1], v2
7948; GFX7-NEXT:    s_endpgm
7949;
7950; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
7951; GFX10-WGP:       ; %bb.0: ; %entry
7952; GFX10-WGP-NEXT:    s_clause 0x1
7953; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7954; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7955; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7956; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7957; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7958; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7959; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7960; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7961; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7962; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7963; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7964; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7965; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7966; GFX10-WGP-NEXT:    buffer_gl0_inv
7967; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7968; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7969; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7970; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7971; GFX10-WGP-NEXT:    s_endpgm
7972;
7973; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
7974; GFX10-CU:       ; %bb.0: ; %entry
7975; GFX10-CU-NEXT:    s_clause 0x1
7976; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7977; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7978; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7979; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7980; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7981; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7982; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7983; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7984; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7985; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7986; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7987; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7988; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7989; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7990; GFX10-CU-NEXT:    s_endpgm
7991;
7992; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
7993; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7994; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7995; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7996; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7997; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7998; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7999; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8000; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8001; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8002; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8003; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8004; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8005; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8006; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8007; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8008; SKIP-CACHE-INV-NEXT:    s_endpgm
8009;
8010; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
8011; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8012; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8013; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8014; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8015; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8016; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8017; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8018; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8019; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8020; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8021;
8022; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
8023; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8024; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8025; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8026; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8027; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8028; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8029; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8030; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8031; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8032; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8033; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8034; GFX90A-TGSPLIT-NEXT:    s_endpgm
8035    i32* %out, i32 %in, i32 %old) {
8036entry:
8037  %gep = getelementptr i32, i32* %out, i32 4
8038  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic seq_cst
8039  %val0 = extractvalue { i32, i1 } %val, 0
8040  store i32 %val0, i32* %out, align 4
8041  ret void
8042}
8043
8044define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
8045; GFX7-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
8046; GFX7:       ; %bb.0: ; %entry
8047; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8048; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8049; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8050; GFX7-NEXT:    s_add_u32 s4, s0, 16
8051; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8052; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8053; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8054; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8055; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8056; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8057; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8058; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8059; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8060; GFX7-NEXT:    flat_store_dword v[0:1], v2
8061; GFX7-NEXT:    s_endpgm
8062;
8063; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
8064; GFX10-WGP:       ; %bb.0: ; %entry
8065; GFX10-WGP-NEXT:    s_clause 0x1
8066; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8067; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8068; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8069; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
8070; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
8071; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
8072; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8073; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
8074; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8075; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8076; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8077; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8078; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8079; GFX10-WGP-NEXT:    buffer_gl0_inv
8080; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8081; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8082; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8083; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8084; GFX10-WGP-NEXT:    s_endpgm
8085;
8086; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
8087; GFX10-CU:       ; %bb.0: ; %entry
8088; GFX10-CU-NEXT:    s_clause 0x1
8089; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8090; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8091; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8092; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
8093; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
8094; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
8095; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8096; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
8097; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8098; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8099; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8100; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8101; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8102; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8103; GFX10-CU-NEXT:    s_endpgm
8104;
8105; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
8106; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8107; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8108; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8109; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8110; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
8111; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
8112; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8113; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8114; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8115; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8116; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8117; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8118; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8119; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8120; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8121; SKIP-CACHE-INV-NEXT:    s_endpgm
8122;
8123; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
8124; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8125; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8126; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8127; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8128; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8129; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8130; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8131; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8132; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8133; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8134;
8135; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
8136; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8137; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8138; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8139; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8140; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8141; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8142; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8143; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8144; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8145; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8146; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8147; GFX90A-TGSPLIT-NEXT:    s_endpgm
8148    i32* %out, i32 %in, i32 %old) {
8149entry:
8150  %gep = getelementptr i32, i32* %out, i32 4
8151  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire seq_cst
8152  %val0 = extractvalue { i32, i1 } %val, 0
8153  store i32 %val0, i32* %out, align 4
8154  ret void
8155}
8156
8157define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_ret_cmpxchg(
8158; GFX7-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
8159; GFX7:       ; %bb.0: ; %entry
8160; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8161; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8162; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8163; GFX7-NEXT:    s_add_u32 s4, s0, 16
8164; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8165; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8166; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8167; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8168; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8169; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8170; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8171; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8172; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8173; GFX7-NEXT:    flat_store_dword v[0:1], v2
8174; GFX7-NEXT:    s_endpgm
8175;
8176; GFX10-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
8177; GFX10-WGP:       ; %bb.0: ; %entry
8178; GFX10-WGP-NEXT:    s_clause 0x1
8179; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8180; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8181; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8182; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
8183; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
8184; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
8185; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8186; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
8187; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8188; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8189; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8190; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8191; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8192; GFX10-WGP-NEXT:    buffer_gl0_inv
8193; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8194; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8195; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8196; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8197; GFX10-WGP-NEXT:    s_endpgm
8198;
8199; GFX10-CU-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
8200; GFX10-CU:       ; %bb.0: ; %entry
8201; GFX10-CU-NEXT:    s_clause 0x1
8202; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8203; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8204; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8205; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
8206; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
8207; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
8208; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8209; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
8210; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8211; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8212; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8213; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8214; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8215; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8216; GFX10-CU-NEXT:    s_endpgm
8217;
8218; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
8219; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8220; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8221; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8222; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8223; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
8224; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
8225; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8226; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8227; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8228; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8229; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8230; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8231; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8232; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8233; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8234; SKIP-CACHE-INV-NEXT:    s_endpgm
8235;
8236; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
8237; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8238; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8239; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8240; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8241; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8242; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8243; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8244; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8245; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8246; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8247;
8248; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
8249; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8250; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8251; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8252; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8253; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8254; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8255; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8256; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8257; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8258; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8259; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8260; GFX90A-TGSPLIT-NEXT:    s_endpgm
8261    i32* %out, i32 %in, i32 %old) {
8262entry:
8263  %gep = getelementptr i32, i32* %out, i32 4
8264  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release seq_cst
8265  %val0 = extractvalue { i32, i1 } %val, 0
8266  store i32 %val0, i32* %out, align 4
8267  ret void
8268}
8269
8270define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
8271; GFX7-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
8272; GFX7:       ; %bb.0: ; %entry
8273; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8274; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8275; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8276; GFX7-NEXT:    s_add_u32 s4, s0, 16
8277; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8278; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8279; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8280; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8281; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8282; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8283; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8284; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8285; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8286; GFX7-NEXT:    flat_store_dword v[0:1], v2
8287; GFX7-NEXT:    s_endpgm
8288;
8289; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
8290; GFX10-WGP:       ; %bb.0: ; %entry
8291; GFX10-WGP-NEXT:    s_clause 0x1
8292; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8293; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8294; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8295; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
8296; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
8297; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
8298; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8299; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
8300; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8301; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8302; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8303; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8304; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8305; GFX10-WGP-NEXT:    buffer_gl0_inv
8306; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8307; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8308; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8309; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8310; GFX10-WGP-NEXT:    s_endpgm
8311;
8312; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
8313; GFX10-CU:       ; %bb.0: ; %entry
8314; GFX10-CU-NEXT:    s_clause 0x1
8315; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8316; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8317; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8318; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
8319; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
8320; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
8321; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8322; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
8323; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8324; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8325; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8326; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8327; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8328; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8329; GFX10-CU-NEXT:    s_endpgm
8330;
8331; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
8332; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8333; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8334; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8335; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8336; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
8337; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
8338; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8339; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8340; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8341; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8342; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8343; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8344; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8345; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8346; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8347; SKIP-CACHE-INV-NEXT:    s_endpgm
8348;
8349; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
8350; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8351; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8352; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8353; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8354; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8355; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8356; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8357; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8358; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8359; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8360;
8361; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
8362; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8363; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8364; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8365; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8366; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8367; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8368; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8369; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8370; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8371; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8372; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8373; GFX90A-TGSPLIT-NEXT:    s_endpgm
8374    i32* %out, i32 %in, i32 %old) {
8375entry:
8376  %gep = getelementptr i32, i32* %out, i32 4
8377  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel seq_cst
8378  %val0 = extractvalue { i32, i1 } %val, 0
8379  store i32 %val0, i32* %out, align 4
8380  ret void
8381}
8382
8383define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
8384; GFX7-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
8385; GFX7:       ; %bb.0: ; %entry
8386; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8387; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8388; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8389; GFX7-NEXT:    s_add_u32 s4, s0, 16
8390; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8391; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8392; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8393; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8394; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8395; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8396; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8397; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8398; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8399; GFX7-NEXT:    flat_store_dword v[0:1], v2
8400; GFX7-NEXT:    s_endpgm
8401;
8402; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
8403; GFX10-WGP:       ; %bb.0: ; %entry
8404; GFX10-WGP-NEXT:    s_clause 0x1
8405; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8406; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8407; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8408; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
8409; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
8410; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
8411; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8412; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
8413; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8414; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8415; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8416; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8417; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8418; GFX10-WGP-NEXT:    buffer_gl0_inv
8419; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8420; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8421; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8422; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8423; GFX10-WGP-NEXT:    s_endpgm
8424;
8425; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
8426; GFX10-CU:       ; %bb.0: ; %entry
8427; GFX10-CU-NEXT:    s_clause 0x1
8428; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8429; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8430; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8431; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
8432; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
8433; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
8434; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8435; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
8436; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8437; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8438; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8439; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8440; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8441; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8442; GFX10-CU-NEXT:    s_endpgm
8443;
8444; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
8445; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8446; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8447; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8448; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8449; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
8450; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
8451; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8452; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8453; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8454; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8455; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8456; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8457; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8458; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8459; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8460; SKIP-CACHE-INV-NEXT:    s_endpgm
8461;
8462; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
8463; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8464; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8465; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8466; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8467; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8468; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8469; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8470; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8471; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8472; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8473;
8474; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
8475; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8476; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8477; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8478; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8479; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8480; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8481; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8482; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8483; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8484; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8485; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8486; GFX90A-TGSPLIT-NEXT:    s_endpgm
8487    i32* %out, i32 %in, i32 %old) {
8488entry:
8489  %gep = getelementptr i32, i32* %out, i32 4
8490  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
8491  %val0 = extractvalue { i32, i1 } %val, 0
8492  store i32 %val0, i32* %out, align 4
8493  ret void
8494}
8495
8496