1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
8; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
9; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
10; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-WGP %s
11; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-CU %s
12
13define amdgpu_kernel void @flat_workgroup_unordered_load(
14; GFX7-LABEL: flat_workgroup_unordered_load:
15; GFX7:       ; %bb.0: ; %entry
16; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
17; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
18; GFX7-NEXT:    v_mov_b32_e32 v0, s0
19; GFX7-NEXT:    v_mov_b32_e32 v1, s1
20; GFX7-NEXT:    flat_load_dword v2, v[0:1]
21; GFX7-NEXT:    v_mov_b32_e32 v0, s2
22; GFX7-NEXT:    v_mov_b32_e32 v1, s3
23; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
24; GFX7-NEXT:    flat_store_dword v[0:1], v2
25; GFX7-NEXT:    s_endpgm
26;
27; GFX10-WGP-LABEL: flat_workgroup_unordered_load:
28; GFX10-WGP:       ; %bb.0: ; %entry
29; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
30; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
31; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
32; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
33; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
34; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
35; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
36; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
37; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
38; GFX10-WGP-NEXT:    s_endpgm
39;
40; GFX10-CU-LABEL: flat_workgroup_unordered_load:
41; GFX10-CU:       ; %bb.0: ; %entry
42; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
43; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
44; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
45; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
46; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
47; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
48; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
49; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
50; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
51; GFX10-CU-NEXT:    s_endpgm
52;
53; SKIP-CACHE-INV-LABEL: flat_workgroup_unordered_load:
54; SKIP-CACHE-INV:       ; %bb.0: ; %entry
55; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
56; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
57; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
58; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
59; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
60; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
61; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
62; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
63; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
64; SKIP-CACHE-INV-NEXT:    s_endpgm
65;
66; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_unordered_load:
67; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
68; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
69; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
70; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
71; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
72; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
73; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
74; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
75; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
76; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
77; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
78;
79; GFX90A-TGSPLIT-LABEL: flat_workgroup_unordered_load:
80; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
81; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
82; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
83; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
84; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
85; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
86; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
87; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
88; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
89; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
90; GFX90A-TGSPLIT-NEXT:    s_endpgm
91;
92; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_unordered_load:
93; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
94; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
95; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
96; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
97; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
98; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
99; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
100; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
101; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
102; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
103; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
104;
105; GFX940-TGSPLIT-LABEL: flat_workgroup_unordered_load:
106; GFX940-TGSPLIT:       ; %bb.0: ; %entry
107; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
108; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
109; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
110; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
111; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
112; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
113; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
114; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
115; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
116; GFX940-TGSPLIT-NEXT:    s_endpgm
117;
118; GFX11-WGP-LABEL: flat_workgroup_unordered_load:
119; GFX11-WGP:       ; %bb.0: ; %entry
120; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
121; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
122; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
123; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
124; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
125; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
126; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
127; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
128; GFX11-WGP-NEXT:    s_endpgm
129;
130; GFX11-CU-LABEL: flat_workgroup_unordered_load:
131; GFX11-CU:       ; %bb.0: ; %entry
132; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
133; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
134; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
135; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
136; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
137; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
138; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
139; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
140; GFX11-CU-NEXT:    s_endpgm
141    i32* %in, i32* %out) {
142entry:
143  %val = load atomic i32, i32* %in syncscope("workgroup") unordered, align 4
144  store i32 %val, i32* %out
145  ret void
146}
147
148define amdgpu_kernel void @flat_workgroup_monotonic_load(
149; GFX7-LABEL: flat_workgroup_monotonic_load:
150; GFX7:       ; %bb.0: ; %entry
151; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
152; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
153; GFX7-NEXT:    v_mov_b32_e32 v0, s0
154; GFX7-NEXT:    v_mov_b32_e32 v1, s1
155; GFX7-NEXT:    flat_load_dword v2, v[0:1]
156; GFX7-NEXT:    v_mov_b32_e32 v0, s2
157; GFX7-NEXT:    v_mov_b32_e32 v1, s3
158; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
159; GFX7-NEXT:    flat_store_dword v[0:1], v2
160; GFX7-NEXT:    s_endpgm
161;
162; GFX10-WGP-LABEL: flat_workgroup_monotonic_load:
163; GFX10-WGP:       ; %bb.0: ; %entry
164; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
165; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
166; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
167; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
168; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
169; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
170; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
171; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
172; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
173; GFX10-WGP-NEXT:    s_endpgm
174;
175; GFX10-CU-LABEL: flat_workgroup_monotonic_load:
176; GFX10-CU:       ; %bb.0: ; %entry
177; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
178; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
179; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
180; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
181; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
182; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
183; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
184; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
185; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
186; GFX10-CU-NEXT:    s_endpgm
187;
188; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_load:
189; SKIP-CACHE-INV:       ; %bb.0: ; %entry
190; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
191; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
192; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
193; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
194; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
195; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
196; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
197; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
198; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
199; SKIP-CACHE-INV-NEXT:    s_endpgm
200;
201; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_load:
202; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
203; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
204; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
205; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
206; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
207; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
208; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
209; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
210; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
211; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
212; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
213;
214; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_load:
215; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
216; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
217; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
218; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
219; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
220; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
221; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
222; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
223; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
224; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
225; GFX90A-TGSPLIT-NEXT:    s_endpgm
226;
227; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_load:
228; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
229; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
230; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
231; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
232; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
233; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
234; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
235; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
236; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
237; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
238; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
239;
240; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_load:
241; GFX940-TGSPLIT:       ; %bb.0: ; %entry
242; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
243; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
244; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
245; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
246; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
247; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
248; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
249; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
250; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
251; GFX940-TGSPLIT-NEXT:    s_endpgm
252;
253; GFX11-WGP-LABEL: flat_workgroup_monotonic_load:
254; GFX11-WGP:       ; %bb.0: ; %entry
255; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
256; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
257; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
258; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
259; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
260; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
261; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
262; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
263; GFX11-WGP-NEXT:    s_endpgm
264;
265; GFX11-CU-LABEL: flat_workgroup_monotonic_load:
266; GFX11-CU:       ; %bb.0: ; %entry
267; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
268; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
269; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
270; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
271; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
272; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
273; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
274; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
275; GFX11-CU-NEXT:    s_endpgm
276    i32* %in, i32* %out) {
277entry:
278  %val = load atomic i32, i32* %in syncscope("workgroup") monotonic, align 4
279  store i32 %val, i32* %out
280  ret void
281}
282
283define amdgpu_kernel void @flat_workgroup_acquire_load(
284; GFX7-LABEL: flat_workgroup_acquire_load:
285; GFX7:       ; %bb.0: ; %entry
286; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
287; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
288; GFX7-NEXT:    v_mov_b32_e32 v0, s0
289; GFX7-NEXT:    v_mov_b32_e32 v1, s1
290; GFX7-NEXT:    flat_load_dword v2, v[0:1]
291; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
292; GFX7-NEXT:    v_mov_b32_e32 v0, s2
293; GFX7-NEXT:    v_mov_b32_e32 v1, s3
294; GFX7-NEXT:    s_waitcnt vmcnt(0)
295; GFX7-NEXT:    flat_store_dword v[0:1], v2
296; GFX7-NEXT:    s_endpgm
297;
298; GFX10-WGP-LABEL: flat_workgroup_acquire_load:
299; GFX10-WGP:       ; %bb.0: ; %entry
300; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
301; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
302; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
303; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
304; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
305; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
306; GFX10-WGP-NEXT:    buffer_gl0_inv
307; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
308; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
309; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
310; GFX10-WGP-NEXT:    s_endpgm
311;
312; GFX10-CU-LABEL: flat_workgroup_acquire_load:
313; GFX10-CU:       ; %bb.0: ; %entry
314; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
315; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
316; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
317; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
318; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
319; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
320; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
321; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
322; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
323; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
324; GFX10-CU-NEXT:    s_endpgm
325;
326; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_load:
327; SKIP-CACHE-INV:       ; %bb.0: ; %entry
328; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
329; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
330; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
331; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
332; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
333; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
334; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
335; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
336; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
337; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
338; SKIP-CACHE-INV-NEXT:    s_endpgm
339;
340; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_load:
341; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
342; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
343; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
344; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
345; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
346; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
347; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
348; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
349; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
350; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
351; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
352; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
353;
354; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_load:
355; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
356; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
357; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
358; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
359; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
360; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
361; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
362; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
363; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
364; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
365; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
366; GFX90A-TGSPLIT-NEXT:    s_endpgm
367;
368; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_load:
369; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
370; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
371; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
372; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
373; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
374; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
375; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
376; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
377; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
378; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
379; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
380; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
381;
382; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_load:
383; GFX940-TGSPLIT:       ; %bb.0: ; %entry
384; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
385; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
386; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
387; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
388; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
389; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
390; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
391; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
392; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
393; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
394; GFX940-TGSPLIT-NEXT:    s_endpgm
395;
396; GFX11-WGP-LABEL: flat_workgroup_acquire_load:
397; GFX11-WGP:       ; %bb.0: ; %entry
398; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
399; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
400; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
401; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
402; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
403; GFX11-WGP-NEXT:    buffer_gl0_inv
404; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
405; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
406; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
407; GFX11-WGP-NEXT:    s_endpgm
408;
409; GFX11-CU-LABEL: flat_workgroup_acquire_load:
410; GFX11-CU:       ; %bb.0: ; %entry
411; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
412; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
413; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
414; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
415; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
416; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
417; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
418; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
419; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
420; GFX11-CU-NEXT:    s_endpgm
421    i32* %in, i32* %out) {
422entry:
423  %val = load atomic i32, i32* %in syncscope("workgroup") acquire, align 4
424  store i32 %val, i32* %out
425  ret void
426}
427
428define amdgpu_kernel void @flat_workgroup_seq_cst_load(
429; GFX7-LABEL: flat_workgroup_seq_cst_load:
430; GFX7:       ; %bb.0: ; %entry
431; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
432; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
433; GFX7-NEXT:    v_mov_b32_e32 v0, s0
434; GFX7-NEXT:    v_mov_b32_e32 v1, s1
435; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
436; GFX7-NEXT:    flat_load_dword v2, v[0:1]
437; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
438; GFX7-NEXT:    v_mov_b32_e32 v0, s2
439; GFX7-NEXT:    v_mov_b32_e32 v1, s3
440; GFX7-NEXT:    s_waitcnt vmcnt(0)
441; GFX7-NEXT:    flat_store_dword v[0:1], v2
442; GFX7-NEXT:    s_endpgm
443;
444; GFX10-WGP-LABEL: flat_workgroup_seq_cst_load:
445; GFX10-WGP:       ; %bb.0: ; %entry
446; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
447; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
448; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
449; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
450; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
451; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
452; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
453; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
454; GFX10-WGP-NEXT:    buffer_gl0_inv
455; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
456; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
457; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
458; GFX10-WGP-NEXT:    s_endpgm
459;
460; GFX10-CU-LABEL: flat_workgroup_seq_cst_load:
461; GFX10-CU:       ; %bb.0: ; %entry
462; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
463; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
464; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
465; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
466; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
467; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
468; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
469; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
470; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
471; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
472; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
473; GFX10-CU-NEXT:    s_endpgm
474;
475; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_load:
476; SKIP-CACHE-INV:       ; %bb.0: ; %entry
477; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
478; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
479; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
480; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
481; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
482; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
483; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
484; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
485; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
486; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
487; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
488; SKIP-CACHE-INV-NEXT:    s_endpgm
489;
490; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_load:
491; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
492; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
493; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
494; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
495; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
496; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
497; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
498; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
499; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
500; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
501; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
502; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
503; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
504;
505; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_load:
506; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
507; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
508; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
509; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
510; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
511; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
512; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
513; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
514; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
515; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
516; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
517; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
518; GFX90A-TGSPLIT-NEXT:    s_endpgm
519;
520; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_load:
521; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
522; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
523; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
524; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
525; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
526; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
527; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
528; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
529; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
530; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
531; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
532; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
533; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
534;
535; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_load:
536; GFX940-TGSPLIT:       ; %bb.0: ; %entry
537; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
538; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
539; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
540; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
541; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
542; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
543; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
544; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
545; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
546; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
547; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
548; GFX940-TGSPLIT-NEXT:    s_endpgm
549;
550; GFX11-WGP-LABEL: flat_workgroup_seq_cst_load:
551; GFX11-WGP:       ; %bb.0: ; %entry
552; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
553; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
554; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
555; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
556; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
557; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
558; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
559; GFX11-WGP-NEXT:    buffer_gl0_inv
560; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
561; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
562; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
563; GFX11-WGP-NEXT:    s_endpgm
564;
565; GFX11-CU-LABEL: flat_workgroup_seq_cst_load:
566; GFX11-CU:       ; %bb.0: ; %entry
567; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
568; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
569; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
570; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
571; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
572; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
573; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
574; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
575; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
576; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
577; GFX11-CU-NEXT:    s_endpgm
578    i32* %in, i32* %out) {
579entry:
580  %val = load atomic i32, i32* %in syncscope("workgroup") seq_cst, align 4
581  store i32 %val, i32* %out
582  ret void
583}
584
585define amdgpu_kernel void @flat_workgroup_unordered_store(
586; GFX7-LABEL: flat_workgroup_unordered_store:
587; GFX7:       ; %bb.0: ; %entry
588; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
589; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
590; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
591; GFX7-NEXT:    v_mov_b32_e32 v0, s0
592; GFX7-NEXT:    v_mov_b32_e32 v1, s1
593; GFX7-NEXT:    v_mov_b32_e32 v2, s2
594; GFX7-NEXT:    flat_store_dword v[0:1], v2
595; GFX7-NEXT:    s_endpgm
596;
597; GFX10-WGP-LABEL: flat_workgroup_unordered_store:
598; GFX10-WGP:       ; %bb.0: ; %entry
599; GFX10-WGP-NEXT:    s_clause 0x1
600; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
601; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
602; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
603; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
604; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
605; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
606; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
607; GFX10-WGP-NEXT:    s_endpgm
608;
609; GFX10-CU-LABEL: flat_workgroup_unordered_store:
610; GFX10-CU:       ; %bb.0: ; %entry
611; GFX10-CU-NEXT:    s_clause 0x1
612; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
613; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
614; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
615; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
616; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
617; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
618; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
619; GFX10-CU-NEXT:    s_endpgm
620;
621; SKIP-CACHE-INV-LABEL: flat_workgroup_unordered_store:
622; SKIP-CACHE-INV:       ; %bb.0: ; %entry
623; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
624; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
625; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
626; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
627; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
628; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
629; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
630; SKIP-CACHE-INV-NEXT:    s_endpgm
631;
632; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_unordered_store:
633; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
634; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
635; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
636; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
637; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
638; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
639; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
640; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
641;
642; GFX90A-TGSPLIT-LABEL: flat_workgroup_unordered_store:
643; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
644; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
645; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
646; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
647; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
648; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
649; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
650; GFX90A-TGSPLIT-NEXT:    s_endpgm
651;
652; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_unordered_store:
653; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
654; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
655; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
656; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
657; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
658; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
659; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
660; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
661;
662; GFX940-TGSPLIT-LABEL: flat_workgroup_unordered_store:
663; GFX940-TGSPLIT:       ; %bb.0: ; %entry
664; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
665; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
666; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
667; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
668; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
669; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
670; GFX940-TGSPLIT-NEXT:    s_endpgm
671;
672; GFX11-WGP-LABEL: flat_workgroup_unordered_store:
673; GFX11-WGP:       ; %bb.0: ; %entry
674; GFX11-WGP-NEXT:    s_clause 0x1
675; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
676; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
677; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
678; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
679; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
680; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
681; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
682; GFX11-WGP-NEXT:    s_endpgm
683;
684; GFX11-CU-LABEL: flat_workgroup_unordered_store:
685; GFX11-CU:       ; %bb.0: ; %entry
686; GFX11-CU-NEXT:    s_clause 0x1
687; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
688; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
689; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
690; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
691; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
692; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
693; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
694; GFX11-CU-NEXT:    s_endpgm
695    i32 %in, i32* %out) {
696entry:
697  store atomic i32 %in, i32* %out syncscope("workgroup") unordered, align 4
698  ret void
699}
700
701define amdgpu_kernel void @flat_workgroup_monotonic_store(
702; GFX7-LABEL: flat_workgroup_monotonic_store:
703; GFX7:       ; %bb.0: ; %entry
704; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
705; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
706; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
707; GFX7-NEXT:    v_mov_b32_e32 v0, s0
708; GFX7-NEXT:    v_mov_b32_e32 v1, s1
709; GFX7-NEXT:    v_mov_b32_e32 v2, s2
710; GFX7-NEXT:    flat_store_dword v[0:1], v2
711; GFX7-NEXT:    s_endpgm
712;
713; GFX10-WGP-LABEL: flat_workgroup_monotonic_store:
714; GFX10-WGP:       ; %bb.0: ; %entry
715; GFX10-WGP-NEXT:    s_clause 0x1
716; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
717; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
718; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
719; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
720; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
721; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
722; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
723; GFX10-WGP-NEXT:    s_endpgm
724;
725; GFX10-CU-LABEL: flat_workgroup_monotonic_store:
726; GFX10-CU:       ; %bb.0: ; %entry
727; GFX10-CU-NEXT:    s_clause 0x1
728; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
729; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
730; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
731; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
732; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
733; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
734; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
735; GFX10-CU-NEXT:    s_endpgm
736;
737; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_store:
738; SKIP-CACHE-INV:       ; %bb.0: ; %entry
739; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
740; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
741; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
742; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
743; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
744; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
745; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
746; SKIP-CACHE-INV-NEXT:    s_endpgm
747;
748; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_store:
749; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
750; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
751; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
752; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
753; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
754; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
755; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
756; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
757;
758; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_store:
759; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
760; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
761; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
762; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
763; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
764; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
765; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
766; GFX90A-TGSPLIT-NEXT:    s_endpgm
767;
768; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_store:
769; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
770; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
771; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
772; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
773; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
774; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
775; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
776; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
777;
778; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_store:
779; GFX940-TGSPLIT:       ; %bb.0: ; %entry
780; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
781; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
782; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
783; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
784; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
785; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
786; GFX940-TGSPLIT-NEXT:    s_endpgm
787;
788; GFX11-WGP-LABEL: flat_workgroup_monotonic_store:
789; GFX11-WGP:       ; %bb.0: ; %entry
790; GFX11-WGP-NEXT:    s_clause 0x1
791; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
792; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
793; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
794; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
795; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
796; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
797; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
798; GFX11-WGP-NEXT:    s_endpgm
799;
800; GFX11-CU-LABEL: flat_workgroup_monotonic_store:
801; GFX11-CU:       ; %bb.0: ; %entry
802; GFX11-CU-NEXT:    s_clause 0x1
803; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
804; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
805; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
806; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
807; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
808; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
809; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
810; GFX11-CU-NEXT:    s_endpgm
811    i32 %in, i32* %out) {
812entry:
813  store atomic i32 %in, i32* %out syncscope("workgroup") monotonic, align 4
814  ret void
815}
816
817define amdgpu_kernel void @flat_workgroup_release_store(
818; GFX7-LABEL: flat_workgroup_release_store:
819; GFX7:       ; %bb.0: ; %entry
820; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
821; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
822; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
823; GFX7-NEXT:    v_mov_b32_e32 v0, s0
824; GFX7-NEXT:    v_mov_b32_e32 v1, s1
825; GFX7-NEXT:    v_mov_b32_e32 v2, s2
826; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
827; GFX7-NEXT:    flat_store_dword v[0:1], v2
828; GFX7-NEXT:    s_endpgm
829;
830; GFX10-WGP-LABEL: flat_workgroup_release_store:
831; GFX10-WGP:       ; %bb.0: ; %entry
832; GFX10-WGP-NEXT:    s_clause 0x1
833; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
834; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
835; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
836; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
837; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
838; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
839; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
840; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
841; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
842; GFX10-WGP-NEXT:    s_endpgm
843;
844; GFX10-CU-LABEL: flat_workgroup_release_store:
845; GFX10-CU:       ; %bb.0: ; %entry
846; GFX10-CU-NEXT:    s_clause 0x1
847; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
848; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
849; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
850; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
851; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
852; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
853; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
854; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
855; GFX10-CU-NEXT:    s_endpgm
856;
857; SKIP-CACHE-INV-LABEL: flat_workgroup_release_store:
858; SKIP-CACHE-INV:       ; %bb.0: ; %entry
859; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
860; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
861; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
862; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
863; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
864; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
865; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
866; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
867; SKIP-CACHE-INV-NEXT:    s_endpgm
868;
869; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_store:
870; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
871; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
872; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
873; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
874; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
875; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
876; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
877; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
878; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
879;
880; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_store:
881; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
882; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
883; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
884; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
885; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
886; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
887; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
888; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
889; GFX90A-TGSPLIT-NEXT:    s_endpgm
890;
891; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_store:
892; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
893; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
894; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
895; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
896; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
897; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
898; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
899; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
900; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
901;
902; GFX940-TGSPLIT-LABEL: flat_workgroup_release_store:
903; GFX940-TGSPLIT:       ; %bb.0: ; %entry
904; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
905; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
906; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
907; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
908; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
909; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
910; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
911; GFX940-TGSPLIT-NEXT:    s_endpgm
912;
913; GFX11-WGP-LABEL: flat_workgroup_release_store:
914; GFX11-WGP:       ; %bb.0: ; %entry
915; GFX11-WGP-NEXT:    s_clause 0x1
916; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
917; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
918; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
919; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
920; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
921; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
922; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
923; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
924; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
925; GFX11-WGP-NEXT:    s_endpgm
926;
927; GFX11-CU-LABEL: flat_workgroup_release_store:
928; GFX11-CU:       ; %bb.0: ; %entry
929; GFX11-CU-NEXT:    s_clause 0x1
930; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
931; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
932; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
933; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
934; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
935; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
936; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
937; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
938; GFX11-CU-NEXT:    s_endpgm
939    i32 %in, i32* %out) {
940entry:
941  store atomic i32 %in, i32* %out syncscope("workgroup") release, align 4
942  ret void
943}
944
945define amdgpu_kernel void @flat_workgroup_seq_cst_store(
946; GFX7-LABEL: flat_workgroup_seq_cst_store:
947; GFX7:       ; %bb.0: ; %entry
948; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
949; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
950; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
951; GFX7-NEXT:    v_mov_b32_e32 v0, s0
952; GFX7-NEXT:    v_mov_b32_e32 v1, s1
953; GFX7-NEXT:    v_mov_b32_e32 v2, s2
954; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
955; GFX7-NEXT:    flat_store_dword v[0:1], v2
956; GFX7-NEXT:    s_endpgm
957;
958; GFX10-WGP-LABEL: flat_workgroup_seq_cst_store:
959; GFX10-WGP:       ; %bb.0: ; %entry
960; GFX10-WGP-NEXT:    s_clause 0x1
961; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
962; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
963; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
964; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
965; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
966; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
967; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
968; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
969; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
970; GFX10-WGP-NEXT:    s_endpgm
971;
972; GFX10-CU-LABEL: flat_workgroup_seq_cst_store:
973; GFX10-CU:       ; %bb.0: ; %entry
974; GFX10-CU-NEXT:    s_clause 0x1
975; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
976; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
977; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
978; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
979; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
980; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
981; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
982; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
983; GFX10-CU-NEXT:    s_endpgm
984;
985; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_store:
986; SKIP-CACHE-INV:       ; %bb.0: ; %entry
987; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
988; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
989; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
990; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
991; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
992; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
993; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
994; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
995; SKIP-CACHE-INV-NEXT:    s_endpgm
996;
997; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_store:
998; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
999; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1000; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
1001; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1002; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1003; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1004; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1005; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1006; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1007;
1008; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_store:
1009; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1010; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1011; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
1012; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1013; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1014; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1015; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1016; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1017; GFX90A-TGSPLIT-NEXT:    s_endpgm
1018;
1019; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_store:
1020; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1021; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
1022; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
1023; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1024; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1025; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1026; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1027; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
1028; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1029;
1030; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_store:
1031; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1032; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
1033; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
1034; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1035; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1036; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1037; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1038; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
1039; GFX940-TGSPLIT-NEXT:    s_endpgm
1040;
1041; GFX11-WGP-LABEL: flat_workgroup_seq_cst_store:
1042; GFX11-WGP:       ; %bb.0: ; %entry
1043; GFX11-WGP-NEXT:    s_clause 0x1
1044; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
1045; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
1046; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1047; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1048; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1049; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1050; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1051; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
1052; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1053; GFX11-WGP-NEXT:    s_endpgm
1054;
1055; GFX11-CU-LABEL: flat_workgroup_seq_cst_store:
1056; GFX11-CU:       ; %bb.0: ; %entry
1057; GFX11-CU-NEXT:    s_clause 0x1
1058; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
1059; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
1060; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1061; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1062; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1063; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1064; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1065; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1066; GFX11-CU-NEXT:    s_endpgm
1067    i32 %in, i32* %out) {
1068entry:
1069  store atomic i32 %in, i32* %out syncscope("workgroup") seq_cst, align 4
1070  ret void
1071}
1072
1073define amdgpu_kernel void @flat_workgroup_monotonic_atomicrmw(
1074; GFX7-LABEL: flat_workgroup_monotonic_atomicrmw:
1075; GFX7:       ; %bb.0: ; %entry
1076; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1077; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1078; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1079; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1080; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1081; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1082; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1083; GFX7-NEXT:    s_endpgm
1084;
1085; GFX10-WGP-LABEL: flat_workgroup_monotonic_atomicrmw:
1086; GFX10-WGP:       ; %bb.0: ; %entry
1087; GFX10-WGP-NEXT:    s_clause 0x1
1088; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1089; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1090; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1091; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1092; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1093; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1094; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1095; GFX10-WGP-NEXT:    s_endpgm
1096;
1097; GFX10-CU-LABEL: flat_workgroup_monotonic_atomicrmw:
1098; GFX10-CU:       ; %bb.0: ; %entry
1099; GFX10-CU-NEXT:    s_clause 0x1
1100; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1101; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1102; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1103; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1104; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1105; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1106; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1107; GFX10-CU-NEXT:    s_endpgm
1108;
1109; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_atomicrmw:
1110; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1111; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1112; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1113; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1114; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1115; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1116; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1117; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1118; SKIP-CACHE-INV-NEXT:    s_endpgm
1119;
1120; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw:
1121; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1122; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1123; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1124; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1125; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1126; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1127; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1128; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1129;
1130; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw:
1131; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1132; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1133; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1134; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1135; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1136; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1137; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1138; GFX90A-TGSPLIT-NEXT:    s_endpgm
1139;
1140; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw:
1141; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1142; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1143; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1144; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1145; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1146; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1147; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1148; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1149;
1150; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw:
1151; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1152; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1153; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1154; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1155; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1156; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1157; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1158; GFX940-TGSPLIT-NEXT:    s_endpgm
1159;
1160; GFX11-WGP-LABEL: flat_workgroup_monotonic_atomicrmw:
1161; GFX11-WGP:       ; %bb.0: ; %entry
1162; GFX11-WGP-NEXT:    s_clause 0x1
1163; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1164; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1165; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1166; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1167; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1168; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1169; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1170; GFX11-WGP-NEXT:    s_endpgm
1171;
1172; GFX11-CU-LABEL: flat_workgroup_monotonic_atomicrmw:
1173; GFX11-CU:       ; %bb.0: ; %entry
1174; GFX11-CU-NEXT:    s_clause 0x1
1175; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1176; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1177; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1178; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1179; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1180; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1181; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1182; GFX11-CU-NEXT:    s_endpgm
1183    i32* %out, i32 %in) {
1184entry:
1185  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") monotonic
1186  ret void
1187}
1188
1189define amdgpu_kernel void @flat_workgroup_acquire_atomicrmw(
1190; GFX7-LABEL: flat_workgroup_acquire_atomicrmw:
1191; GFX7:       ; %bb.0: ; %entry
1192; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1193; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1194; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1195; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1196; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1197; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1198; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1199; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1200; GFX7-NEXT:    s_endpgm
1201;
1202; GFX10-WGP-LABEL: flat_workgroup_acquire_atomicrmw:
1203; GFX10-WGP:       ; %bb.0: ; %entry
1204; GFX10-WGP-NEXT:    s_clause 0x1
1205; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1206; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1207; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1208; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1209; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1210; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1211; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1212; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1213; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1214; GFX10-WGP-NEXT:    buffer_gl0_inv
1215; GFX10-WGP-NEXT:    s_endpgm
1216;
1217; GFX10-CU-LABEL: flat_workgroup_acquire_atomicrmw:
1218; GFX10-CU:       ; %bb.0: ; %entry
1219; GFX10-CU-NEXT:    s_clause 0x1
1220; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1221; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1222; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1223; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1224; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1225; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1226; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1227; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1228; GFX10-CU-NEXT:    s_endpgm
1229;
1230; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_atomicrmw:
1231; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1232; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1233; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1234; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1235; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1236; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1237; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1238; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1239; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1240; SKIP-CACHE-INV-NEXT:    s_endpgm
1241;
1242; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw:
1243; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1244; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1245; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1246; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1247; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1248; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1249; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1250; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1251; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1252;
1253; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw:
1254; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1255; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1256; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1257; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1258; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1259; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1260; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1261; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1262; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1263; GFX90A-TGSPLIT-NEXT:    s_endpgm
1264;
1265; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw:
1266; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1267; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1268; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1269; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1270; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1271; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1272; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1273; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1274; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1275;
1276; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw:
1277; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1278; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1279; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1280; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1281; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1282; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1283; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1284; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1285; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
1286; GFX940-TGSPLIT-NEXT:    s_endpgm
1287;
1288; GFX11-WGP-LABEL: flat_workgroup_acquire_atomicrmw:
1289; GFX11-WGP:       ; %bb.0: ; %entry
1290; GFX11-WGP-NEXT:    s_clause 0x1
1291; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1292; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1293; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1294; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1295; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1296; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1297; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1298; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1299; GFX11-WGP-NEXT:    buffer_gl0_inv
1300; GFX11-WGP-NEXT:    s_endpgm
1301;
1302; GFX11-CU-LABEL: flat_workgroup_acquire_atomicrmw:
1303; GFX11-CU:       ; %bb.0: ; %entry
1304; GFX11-CU-NEXT:    s_clause 0x1
1305; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1306; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1307; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1308; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1309; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1310; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1311; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1312; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1313; GFX11-CU-NEXT:    s_endpgm
1314    i32* %out, i32 %in) {
1315entry:
1316  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acquire
1317  ret void
1318}
1319
1320define amdgpu_kernel void @flat_workgroup_release_atomicrmw(
1321; GFX7-LABEL: flat_workgroup_release_atomicrmw:
1322; GFX7:       ; %bb.0: ; %entry
1323; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1324; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1325; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1326; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1327; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1328; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1329; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1330; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1331; GFX7-NEXT:    s_endpgm
1332;
1333; GFX10-WGP-LABEL: flat_workgroup_release_atomicrmw:
1334; GFX10-WGP:       ; %bb.0: ; %entry
1335; GFX10-WGP-NEXT:    s_clause 0x1
1336; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1337; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1338; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1339; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1340; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1341; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1342; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1343; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1344; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1345; GFX10-WGP-NEXT:    s_endpgm
1346;
1347; GFX10-CU-LABEL: flat_workgroup_release_atomicrmw:
1348; GFX10-CU:       ; %bb.0: ; %entry
1349; GFX10-CU-NEXT:    s_clause 0x1
1350; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1351; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1352; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1353; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1354; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1355; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1356; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1357; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1358; GFX10-CU-NEXT:    s_endpgm
1359;
1360; SKIP-CACHE-INV-LABEL: flat_workgroup_release_atomicrmw:
1361; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1362; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1363; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1364; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1365; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1366; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1367; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1368; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1369; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1370; SKIP-CACHE-INV-NEXT:    s_endpgm
1371;
1372; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_atomicrmw:
1373; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1374; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1375; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1376; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1377; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1378; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1379; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1380; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1381; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1382;
1383; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_atomicrmw:
1384; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1385; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1386; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1387; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1388; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1389; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1390; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1391; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1392; GFX90A-TGSPLIT-NEXT:    s_endpgm
1393;
1394; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_atomicrmw:
1395; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1396; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1397; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1398; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1399; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1400; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1401; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1402; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1403; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1404;
1405; GFX940-TGSPLIT-LABEL: flat_workgroup_release_atomicrmw:
1406; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1407; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1408; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1409; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1410; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1411; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1412; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1413; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1414; GFX940-TGSPLIT-NEXT:    s_endpgm
1415;
1416; GFX11-WGP-LABEL: flat_workgroup_release_atomicrmw:
1417; GFX11-WGP:       ; %bb.0: ; %entry
1418; GFX11-WGP-NEXT:    s_clause 0x1
1419; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1420; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1421; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1422; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1423; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1424; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1425; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1426; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1427; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1428; GFX11-WGP-NEXT:    s_endpgm
1429;
1430; GFX11-CU-LABEL: flat_workgroup_release_atomicrmw:
1431; GFX11-CU:       ; %bb.0: ; %entry
1432; GFX11-CU-NEXT:    s_clause 0x1
1433; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1434; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1435; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1436; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1437; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1438; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1439; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1440; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1441; GFX11-CU-NEXT:    s_endpgm
1442    i32* %out, i32 %in) {
1443entry:
1444  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") release
1445  ret void
1446}
1447
1448define amdgpu_kernel void @flat_workgroup_acq_rel_atomicrmw(
1449; GFX7-LABEL: flat_workgroup_acq_rel_atomicrmw:
1450; GFX7:       ; %bb.0: ; %entry
1451; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1452; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1453; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1454; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1455; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1456; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1457; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1458; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1459; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1460; GFX7-NEXT:    s_endpgm
1461;
1462; GFX10-WGP-LABEL: flat_workgroup_acq_rel_atomicrmw:
1463; GFX10-WGP:       ; %bb.0: ; %entry
1464; GFX10-WGP-NEXT:    s_clause 0x1
1465; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1466; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1467; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1468; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1469; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1470; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1471; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1472; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1473; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1474; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1475; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1476; GFX10-WGP-NEXT:    buffer_gl0_inv
1477; GFX10-WGP-NEXT:    s_endpgm
1478;
1479; GFX10-CU-LABEL: flat_workgroup_acq_rel_atomicrmw:
1480; GFX10-CU:       ; %bb.0: ; %entry
1481; GFX10-CU-NEXT:    s_clause 0x1
1482; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1483; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1484; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1485; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1486; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1487; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1488; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1489; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1490; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1491; GFX10-CU-NEXT:    s_endpgm
1492;
1493; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_atomicrmw:
1494; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1495; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1496; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1497; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1498; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1499; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1500; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1501; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1502; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1503; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1504; SKIP-CACHE-INV-NEXT:    s_endpgm
1505;
1506; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw:
1507; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1508; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1509; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1510; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1511; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1512; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1513; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1514; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1515; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1516; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1517;
1518; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw:
1519; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1520; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1521; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1522; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1523; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1524; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1525; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1526; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1527; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1528; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1529; GFX90A-TGSPLIT-NEXT:    s_endpgm
1530;
1531; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw:
1532; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1533; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1534; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1535; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1536; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1537; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1538; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1539; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1540; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1541; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1542;
1543; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw:
1544; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1545; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1546; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1547; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1548; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1549; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1550; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1551; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1552; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1553; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
1554; GFX940-TGSPLIT-NEXT:    s_endpgm
1555;
1556; GFX11-WGP-LABEL: flat_workgroup_acq_rel_atomicrmw:
1557; GFX11-WGP:       ; %bb.0: ; %entry
1558; GFX11-WGP-NEXT:    s_clause 0x1
1559; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1560; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1561; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1562; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1563; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1564; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1565; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1566; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1567; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1568; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1569; GFX11-WGP-NEXT:    buffer_gl0_inv
1570; GFX11-WGP-NEXT:    s_endpgm
1571;
1572; GFX11-CU-LABEL: flat_workgroup_acq_rel_atomicrmw:
1573; GFX11-CU:       ; %bb.0: ; %entry
1574; GFX11-CU-NEXT:    s_clause 0x1
1575; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1576; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1577; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1578; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1579; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1580; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1581; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1582; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1583; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1584; GFX11-CU-NEXT:    s_endpgm
1585    i32* %out, i32 %in) {
1586entry:
1587  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acq_rel
1588  ret void
1589}
1590
1591define amdgpu_kernel void @flat_workgroup_seq_cst_atomicrmw(
1592; GFX7-LABEL: flat_workgroup_seq_cst_atomicrmw:
1593; GFX7:       ; %bb.0: ; %entry
1594; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1595; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1596; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1597; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1598; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1599; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1600; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1601; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1602; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1603; GFX7-NEXT:    s_endpgm
1604;
1605; GFX10-WGP-LABEL: flat_workgroup_seq_cst_atomicrmw:
1606; GFX10-WGP:       ; %bb.0: ; %entry
1607; GFX10-WGP-NEXT:    s_clause 0x1
1608; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1609; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1610; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1611; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1612; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1613; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1614; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1615; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1616; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1617; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1618; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1619; GFX10-WGP-NEXT:    buffer_gl0_inv
1620; GFX10-WGP-NEXT:    s_endpgm
1621;
1622; GFX10-CU-LABEL: flat_workgroup_seq_cst_atomicrmw:
1623; GFX10-CU:       ; %bb.0: ; %entry
1624; GFX10-CU-NEXT:    s_clause 0x1
1625; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1626; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1627; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1628; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1629; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1630; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1631; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1632; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1633; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1634; GFX10-CU-NEXT:    s_endpgm
1635;
1636; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_atomicrmw:
1637; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1638; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1639; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1640; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1641; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1642; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1643; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1644; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1645; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1646; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1647; SKIP-CACHE-INV-NEXT:    s_endpgm
1648;
1649; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw:
1650; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1651; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1652; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1653; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1654; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1655; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1656; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1657; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1658; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1659; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1660;
1661; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw:
1662; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1663; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1664; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1665; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1666; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1667; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1668; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1669; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1670; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1671; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1672; GFX90A-TGSPLIT-NEXT:    s_endpgm
1673;
1674; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw:
1675; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1676; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1677; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1678; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1679; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1680; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1681; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1682; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1683; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1684; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1685;
1686; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw:
1687; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1688; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1689; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1690; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1691; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1692; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1693; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1694; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1695; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1696; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
1697; GFX940-TGSPLIT-NEXT:    s_endpgm
1698;
1699; GFX11-WGP-LABEL: flat_workgroup_seq_cst_atomicrmw:
1700; GFX11-WGP:       ; %bb.0: ; %entry
1701; GFX11-WGP-NEXT:    s_clause 0x1
1702; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1703; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1704; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1705; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1706; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1707; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1708; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1709; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1710; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1711; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1712; GFX11-WGP-NEXT:    buffer_gl0_inv
1713; GFX11-WGP-NEXT:    s_endpgm
1714;
1715; GFX11-CU-LABEL: flat_workgroup_seq_cst_atomicrmw:
1716; GFX11-CU:       ; %bb.0: ; %entry
1717; GFX11-CU-NEXT:    s_clause 0x1
1718; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1719; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1720; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1721; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1722; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1723; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1724; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1725; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1726; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1727; GFX11-CU-NEXT:    s_endpgm
1728    i32* %out, i32 %in) {
1729entry:
1730  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") seq_cst
1731  ret void
1732}
1733
1734define amdgpu_kernel void @flat_workgroup_acquire_ret_atomicrmw(
1735; GFX7-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1736; GFX7:       ; %bb.0: ; %entry
1737; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1738; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1739; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1740; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1741; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1742; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1743; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1744; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1745; GFX7-NEXT:    flat_store_dword v[0:1], v2
1746; GFX7-NEXT:    s_endpgm
1747;
1748; GFX10-WGP-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1749; GFX10-WGP:       ; %bb.0: ; %entry
1750; GFX10-WGP-NEXT:    s_clause 0x1
1751; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1752; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1753; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1754; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1755; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1756; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1757; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1758; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1759; GFX10-WGP-NEXT:    buffer_gl0_inv
1760; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1761; GFX10-WGP-NEXT:    s_endpgm
1762;
1763; GFX10-CU-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1764; GFX10-CU:       ; %bb.0: ; %entry
1765; GFX10-CU-NEXT:    s_clause 0x1
1766; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1767; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1768; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1769; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1770; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1771; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1772; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1773; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1774; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1775; GFX10-CU-NEXT:    s_endpgm
1776;
1777; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1778; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1779; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1780; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1781; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1782; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1783; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1784; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1785; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1786; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1787; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1788; SKIP-CACHE-INV-NEXT:    s_endpgm
1789;
1790; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1791; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1792; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1793; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1794; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1795; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1796; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1797; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1798; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1799; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1800; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1801;
1802; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1803; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1804; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1805; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1806; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1807; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1808; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1809; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1810; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1811; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1812; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1813; GFX90A-TGSPLIT-NEXT:    s_endpgm
1814;
1815; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1816; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1817; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1818; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1819; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1820; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1821; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1822; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1823; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1824; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1825; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1826;
1827; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1828; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1829; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1830; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1831; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1832; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1833; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1834; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1835; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1836; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
1837; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1838; GFX940-TGSPLIT-NEXT:    s_endpgm
1839;
1840; GFX11-WGP-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1841; GFX11-WGP:       ; %bb.0: ; %entry
1842; GFX11-WGP-NEXT:    s_clause 0x1
1843; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1844; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1845; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1846; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1847; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1848; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1849; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1850; GFX11-WGP-NEXT:    buffer_gl0_inv
1851; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
1852; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1853; GFX11-WGP-NEXT:    s_endpgm
1854;
1855; GFX11-CU-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1856; GFX11-CU:       ; %bb.0: ; %entry
1857; GFX11-CU-NEXT:    s_clause 0x1
1858; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1859; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1860; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1861; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1862; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1863; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1864; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1865; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1866; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1867; GFX11-CU-NEXT:    s_endpgm
1868    i32* %out, i32 %in) {
1869entry:
1870  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acquire
1871  store i32 %val, i32* %out, align 4
1872  ret void
1873}
1874
1875define amdgpu_kernel void @flat_workgroup_acq_rel_ret_atomicrmw(
1876; GFX7-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1877; GFX7:       ; %bb.0: ; %entry
1878; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1879; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1880; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1881; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1882; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1883; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1884; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1885; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1886; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1887; GFX7-NEXT:    flat_store_dword v[0:1], v2
1888; GFX7-NEXT:    s_endpgm
1889;
1890; GFX10-WGP-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1891; GFX10-WGP:       ; %bb.0: ; %entry
1892; GFX10-WGP-NEXT:    s_clause 0x1
1893; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1894; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1895; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1896; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1897; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1898; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1899; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1900; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1901; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1902; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1903; GFX10-WGP-NEXT:    buffer_gl0_inv
1904; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1905; GFX10-WGP-NEXT:    s_endpgm
1906;
1907; GFX10-CU-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1908; GFX10-CU:       ; %bb.0: ; %entry
1909; GFX10-CU-NEXT:    s_clause 0x1
1910; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1911; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1912; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1913; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1914; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1915; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1916; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1917; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1918; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1919; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1920; GFX10-CU-NEXT:    s_endpgm
1921;
1922; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1923; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1924; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1925; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1926; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1927; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1928; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1929; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1930; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1931; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1932; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1933; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1934; SKIP-CACHE-INV-NEXT:    s_endpgm
1935;
1936; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1937; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1938; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1939; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1940; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1941; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1942; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1943; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1944; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1945; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1946; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1947; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1948;
1949; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1950; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1951; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1952; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1953; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1954; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1955; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1956; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1957; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1958; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1959; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1960; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1961; GFX90A-TGSPLIT-NEXT:    s_endpgm
1962;
1963; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1964; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1965; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1966; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1967; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1968; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1969; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1970; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1971; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1972; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1973; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1974; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1975;
1976; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1977; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1978; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1979; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1980; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1981; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1982; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1983; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1984; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1985; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1986; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
1987; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1988; GFX940-TGSPLIT-NEXT:    s_endpgm
1989;
1990; GFX11-WGP-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1991; GFX11-WGP:       ; %bb.0: ; %entry
1992; GFX11-WGP-NEXT:    s_clause 0x1
1993; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1994; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1995; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1996; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1997; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1998; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1999; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2000; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2001; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2002; GFX11-WGP-NEXT:    buffer_gl0_inv
2003; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
2004; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2005; GFX11-WGP-NEXT:    s_endpgm
2006;
2007; GFX11-CU-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
2008; GFX11-CU:       ; %bb.0: ; %entry
2009; GFX11-CU-NEXT:    s_clause 0x1
2010; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2011; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
2012; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2013; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2014; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2015; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2016; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2017; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2018; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
2019; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2020; GFX11-CU-NEXT:    s_endpgm
2021    i32* %out, i32 %in) {
2022entry:
2023  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acq_rel
2024  store i32 %val, i32* %out, align 4
2025  ret void
2026}
2027
2028define amdgpu_kernel void @flat_workgroup_seq_cst_ret_atomicrmw(
2029; GFX7-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2030; GFX7:       ; %bb.0: ; %entry
2031; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2032; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
2033; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2034; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2035; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2036; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2037; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2038; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2039; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2040; GFX7-NEXT:    flat_store_dword v[0:1], v2
2041; GFX7-NEXT:    s_endpgm
2042;
2043; GFX10-WGP-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2044; GFX10-WGP:       ; %bb.0: ; %entry
2045; GFX10-WGP-NEXT:    s_clause 0x1
2046; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2047; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
2048; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2049; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2050; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2051; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2052; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2053; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2054; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2055; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2056; GFX10-WGP-NEXT:    buffer_gl0_inv
2057; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2058; GFX10-WGP-NEXT:    s_endpgm
2059;
2060; GFX10-CU-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2061; GFX10-CU:       ; %bb.0: ; %entry
2062; GFX10-CU-NEXT:    s_clause 0x1
2063; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2064; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
2065; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2066; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2067; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2068; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2069; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2070; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2071; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2072; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2073; GFX10-CU-NEXT:    s_endpgm
2074;
2075; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2076; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2077; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2078; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
2079; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2080; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2081; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2082; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2083; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2084; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2085; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2086; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2087; SKIP-CACHE-INV-NEXT:    s_endpgm
2088;
2089; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2090; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2091; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2092; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2093; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2094; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2095; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
2096; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2097; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2098; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2099; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2100; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2101;
2102; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2103; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2104; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2105; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2106; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2107; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2108; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
2109; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2110; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2111; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2112; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2113; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2114; GFX90A-TGSPLIT-NEXT:    s_endpgm
2115;
2116; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2117; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2118; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2119; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2120; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2121; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2122; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2123; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2124; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
2125; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2126; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2127; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2128;
2129; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2130; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2131; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2132; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2133; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2134; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2135; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2136; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2137; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
2138; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2139; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
2140; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2141; GFX940-TGSPLIT-NEXT:    s_endpgm
2142;
2143; GFX11-WGP-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2144; GFX11-WGP:       ; %bb.0: ; %entry
2145; GFX11-WGP-NEXT:    s_clause 0x1
2146; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2147; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
2148; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2149; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2150; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2151; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2152; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2153; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2154; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2155; GFX11-WGP-NEXT:    buffer_gl0_inv
2156; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
2157; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2158; GFX11-WGP-NEXT:    s_endpgm
2159;
2160; GFX11-CU-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2161; GFX11-CU:       ; %bb.0: ; %entry
2162; GFX11-CU-NEXT:    s_clause 0x1
2163; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2164; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
2165; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2166; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2167; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2168; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2169; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2170; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2171; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
2172; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2173; GFX11-CU-NEXT:    s_endpgm
2174    i32* %out, i32 %in) {
2175entry:
2176  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") seq_cst
2177  store i32 %val, i32* %out, align 4
2178  ret void
2179}
2180
2181define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_cmpxchg(
2182; GFX7-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2183; GFX7:       ; %bb.0: ; %entry
2184; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2185; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2186; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2187; GFX7-NEXT:    s_add_u32 s0, s0, 16
2188; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2189; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2190; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2191; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2192; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2193; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2194; GFX7-NEXT:    s_endpgm
2195;
2196; GFX10-WGP-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2197; GFX10-WGP:       ; %bb.0: ; %entry
2198; GFX10-WGP-NEXT:    s_clause 0x1
2199; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2200; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2201; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2202; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2203; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2204; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2205; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2206; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2207; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2208; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2209; GFX10-WGP-NEXT:    s_endpgm
2210;
2211; GFX10-CU-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2212; GFX10-CU:       ; %bb.0: ; %entry
2213; GFX10-CU-NEXT:    s_clause 0x1
2214; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2215; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2216; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2217; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2218; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2219; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2220; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2221; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2222; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2223; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2224; GFX10-CU-NEXT:    s_endpgm
2225;
2226; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2227; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2228; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2229; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2230; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2231; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2232; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2233; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2234; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2235; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2236; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2237; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2238; SKIP-CACHE-INV-NEXT:    s_endpgm
2239;
2240; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2241; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2242; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2243; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2244; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2245; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2246; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2247; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2248; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2249;
2250; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2251; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2252; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2253; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2254; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2255; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2256; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2257; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2258; GFX90A-TGSPLIT-NEXT:    s_endpgm
2259;
2260; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2261; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2262; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2263; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2264; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2265; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2266; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2267; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2268; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2269;
2270; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2271; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2272; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2273; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2274; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2275; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2276; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2277; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2278; GFX940-TGSPLIT-NEXT:    s_endpgm
2279;
2280; GFX11-WGP-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2281; GFX11-WGP:       ; %bb.0: ; %entry
2282; GFX11-WGP-NEXT:    s_clause 0x1
2283; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2284; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2285; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2286; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2287; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2288; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2289; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2290; GFX11-WGP-NEXT:    s_endpgm
2291;
2292; GFX11-CU-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2293; GFX11-CU:       ; %bb.0: ; %entry
2294; GFX11-CU-NEXT:    s_clause 0x1
2295; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2296; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2297; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2298; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2299; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2300; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2301; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2302; GFX11-CU-NEXT:    s_endpgm
2303    i32* %out, i32 %in, i32 %old) {
2304entry:
2305  %gep = getelementptr i32, i32* %out, i32 4
2306  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic
2307  ret void
2308}
2309
2310define amdgpu_kernel void @flat_workgroup_acquire_monotonic_cmpxchg(
2311; GFX7-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
2312; GFX7:       ; %bb.0: ; %entry
2313; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2314; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2315; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2316; GFX7-NEXT:    s_add_u32 s0, s0, 16
2317; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2318; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2319; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2320; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2321; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2322; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2323; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2324; GFX7-NEXT:    s_endpgm
2325;
2326; GFX10-WGP-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
2327; GFX10-WGP:       ; %bb.0: ; %entry
2328; GFX10-WGP-NEXT:    s_clause 0x1
2329; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2330; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2331; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2332; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2333; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2334; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2335; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2336; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2337; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2338; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2339; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2340; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2341; GFX10-WGP-NEXT:    buffer_gl0_inv
2342; GFX10-WGP-NEXT:    s_endpgm
2343;
2344; GFX10-CU-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
2345; GFX10-CU:       ; %bb.0: ; %entry
2346; GFX10-CU-NEXT:    s_clause 0x1
2347; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2348; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2349; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2350; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2351; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2352; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2353; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2354; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2355; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2356; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2357; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2358; GFX10-CU-NEXT:    s_endpgm
2359;
2360; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
2361; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2362; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2363; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2364; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2365; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2366; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2367; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2368; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2369; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2370; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2371; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2372; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2373; SKIP-CACHE-INV-NEXT:    s_endpgm
2374;
2375; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
2376; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2377; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2378; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2379; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2380; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2381; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2382; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2383; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2384; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2385;
2386; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
2387; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2388; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2389; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2390; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2391; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2392; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2393; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2394; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2395; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2396; GFX90A-TGSPLIT-NEXT:    s_endpgm
2397;
2398; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
2399; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2400; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2401; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2402; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2403; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2404; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2405; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2406; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2407; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2408;
2409; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
2410; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2411; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2412; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2413; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2414; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2415; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2416; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2417; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2418; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
2419; GFX940-TGSPLIT-NEXT:    s_endpgm
2420;
2421; GFX11-WGP-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
2422; GFX11-WGP:       ; %bb.0: ; %entry
2423; GFX11-WGP-NEXT:    s_clause 0x1
2424; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2425; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2426; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2427; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2428; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2429; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2430; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2431; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2432; GFX11-WGP-NEXT:    buffer_gl0_inv
2433; GFX11-WGP-NEXT:    s_endpgm
2434;
2435; GFX11-CU-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
2436; GFX11-CU:       ; %bb.0: ; %entry
2437; GFX11-CU-NEXT:    s_clause 0x1
2438; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2439; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2440; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2441; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2442; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2443; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2444; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2445; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2446; GFX11-CU-NEXT:    s_endpgm
2447    i32* %out, i32 %in, i32 %old) {
2448entry:
2449  %gep = getelementptr i32, i32* %out, i32 4
2450  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
2451  ret void
2452}
2453
2454define amdgpu_kernel void @flat_workgroup_release_monotonic_cmpxchg(
2455; GFX7-LABEL: flat_workgroup_release_monotonic_cmpxchg:
2456; GFX7:       ; %bb.0: ; %entry
2457; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2458; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2459; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2460; GFX7-NEXT:    s_add_u32 s0, s0, 16
2461; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2462; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2463; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2464; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2465; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2466; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2467; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2468; GFX7-NEXT:    s_endpgm
2469;
2470; GFX10-WGP-LABEL: flat_workgroup_release_monotonic_cmpxchg:
2471; GFX10-WGP:       ; %bb.0: ; %entry
2472; GFX10-WGP-NEXT:    s_clause 0x1
2473; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2474; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2475; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2476; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2477; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2478; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2479; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2480; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2481; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2482; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2483; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2484; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2485; GFX10-WGP-NEXT:    s_endpgm
2486;
2487; GFX10-CU-LABEL: flat_workgroup_release_monotonic_cmpxchg:
2488; GFX10-CU:       ; %bb.0: ; %entry
2489; GFX10-CU-NEXT:    s_clause 0x1
2490; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2491; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2492; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2493; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2494; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2495; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2496; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2497; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2498; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2499; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2500; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2501; GFX10-CU-NEXT:    s_endpgm
2502;
2503; SKIP-CACHE-INV-LABEL: flat_workgroup_release_monotonic_cmpxchg:
2504; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2505; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2506; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2507; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2508; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2509; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2510; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2511; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2512; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2513; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2514; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2515; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2516; SKIP-CACHE-INV-NEXT:    s_endpgm
2517;
2518; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg:
2519; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2520; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2521; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2522; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2523; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2524; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2525; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2526; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2527; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2528;
2529; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg:
2530; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2531; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2532; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2533; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2534; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2535; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2536; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2537; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2538; GFX90A-TGSPLIT-NEXT:    s_endpgm
2539;
2540; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg:
2541; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2542; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2543; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2544; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2545; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2546; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2547; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2548; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2549; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2550;
2551; GFX940-TGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg:
2552; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2553; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2554; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2555; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2556; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2557; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2558; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2559; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2560; GFX940-TGSPLIT-NEXT:    s_endpgm
2561;
2562; GFX11-WGP-LABEL: flat_workgroup_release_monotonic_cmpxchg:
2563; GFX11-WGP:       ; %bb.0: ; %entry
2564; GFX11-WGP-NEXT:    s_clause 0x1
2565; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2566; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2567; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2568; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2569; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2570; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2571; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2572; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2573; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2574; GFX11-WGP-NEXT:    s_endpgm
2575;
2576; GFX11-CU-LABEL: flat_workgroup_release_monotonic_cmpxchg:
2577; GFX11-CU:       ; %bb.0: ; %entry
2578; GFX11-CU-NEXT:    s_clause 0x1
2579; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2580; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2581; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2582; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2583; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2584; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2585; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2586; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2587; GFX11-CU-NEXT:    s_endpgm
2588    i32* %out, i32 %in, i32 %old) {
2589entry:
2590  %gep = getelementptr i32, i32* %out, i32 4
2591  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic
2592  ret void
2593}
2594
2595define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_cmpxchg(
2596; GFX7-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
2597; GFX7:       ; %bb.0: ; %entry
2598; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2599; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2600; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2601; GFX7-NEXT:    s_add_u32 s0, s0, 16
2602; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2603; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2604; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2605; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2606; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2607; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2608; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2609; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2610; GFX7-NEXT:    s_endpgm
2611;
2612; GFX10-WGP-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
2613; GFX10-WGP:       ; %bb.0: ; %entry
2614; GFX10-WGP-NEXT:    s_clause 0x1
2615; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2616; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2617; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2618; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2619; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2620; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2621; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2622; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2623; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2624; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2625; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2626; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2627; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2628; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2629; GFX10-WGP-NEXT:    buffer_gl0_inv
2630; GFX10-WGP-NEXT:    s_endpgm
2631;
2632; GFX10-CU-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
2633; GFX10-CU:       ; %bb.0: ; %entry
2634; GFX10-CU-NEXT:    s_clause 0x1
2635; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2636; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2637; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2638; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2639; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2640; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2641; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2642; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2643; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2644; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2645; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2646; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2647; GFX10-CU-NEXT:    s_endpgm
2648;
2649; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
2650; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2651; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2652; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2653; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2654; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2655; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2656; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2657; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2658; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2659; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2660; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2661; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2662; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2663; SKIP-CACHE-INV-NEXT:    s_endpgm
2664;
2665; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
2666; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2667; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2668; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2669; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2670; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2671; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2672; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2673; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2674; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2675; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2676;
2677; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
2678; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2679; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2680; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2681; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2682; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2683; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2684; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2685; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2686; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2687; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2688; GFX90A-TGSPLIT-NEXT:    s_endpgm
2689;
2690; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
2691; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2692; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2693; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2694; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2695; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2696; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2697; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2698; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2699; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2700; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2701;
2702; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
2703; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2704; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2705; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2706; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2707; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2708; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2709; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2710; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2711; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2712; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
2713; GFX940-TGSPLIT-NEXT:    s_endpgm
2714;
2715; GFX11-WGP-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
2716; GFX11-WGP:       ; %bb.0: ; %entry
2717; GFX11-WGP-NEXT:    s_clause 0x1
2718; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2719; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2720; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2721; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2722; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2723; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2724; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2725; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2726; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2727; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2728; GFX11-WGP-NEXT:    buffer_gl0_inv
2729; GFX11-WGP-NEXT:    s_endpgm
2730;
2731; GFX11-CU-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
2732; GFX11-CU:       ; %bb.0: ; %entry
2733; GFX11-CU-NEXT:    s_clause 0x1
2734; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2735; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2736; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2737; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2738; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2739; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2740; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2741; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2742; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2743; GFX11-CU-NEXT:    s_endpgm
2744    i32* %out, i32 %in, i32 %old) {
2745entry:
2746  %gep = getelementptr i32, i32* %out, i32 4
2747  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
2748  ret void
2749}
2750
2751define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_cmpxchg(
2752; GFX7-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
2753; GFX7:       ; %bb.0: ; %entry
2754; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2755; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2756; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2757; GFX7-NEXT:    s_add_u32 s0, s0, 16
2758; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2759; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2760; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2761; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2762; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2763; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2764; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2765; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2766; GFX7-NEXT:    s_endpgm
2767;
2768; GFX10-WGP-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
2769; GFX10-WGP:       ; %bb.0: ; %entry
2770; GFX10-WGP-NEXT:    s_clause 0x1
2771; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2772; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2773; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2774; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2775; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2776; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2777; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2778; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2779; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2780; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2781; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2782; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2783; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2784; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2785; GFX10-WGP-NEXT:    buffer_gl0_inv
2786; GFX10-WGP-NEXT:    s_endpgm
2787;
2788; GFX10-CU-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
2789; GFX10-CU:       ; %bb.0: ; %entry
2790; GFX10-CU-NEXT:    s_clause 0x1
2791; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2792; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2793; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2794; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2795; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2796; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2797; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2798; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2799; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2800; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2801; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2802; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2803; GFX10-CU-NEXT:    s_endpgm
2804;
2805; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
2806; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2807; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2808; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2809; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2810; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2811; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2812; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2813; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2814; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2815; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2816; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2817; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2818; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2819; SKIP-CACHE-INV-NEXT:    s_endpgm
2820;
2821; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
2822; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2823; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2824; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2825; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2826; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2827; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2828; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2829; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2830; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2831; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2832;
2833; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
2834; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2835; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2836; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2837; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2838; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2839; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2840; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2841; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2842; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2843; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2844; GFX90A-TGSPLIT-NEXT:    s_endpgm
2845;
2846; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
2847; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2848; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2849; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2850; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2851; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2852; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2853; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2854; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2855; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2856; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2857;
2858; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
2859; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2860; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2861; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2862; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2863; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2864; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2865; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2866; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2867; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2868; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
2869; GFX940-TGSPLIT-NEXT:    s_endpgm
2870;
2871; GFX11-WGP-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
2872; GFX11-WGP:       ; %bb.0: ; %entry
2873; GFX11-WGP-NEXT:    s_clause 0x1
2874; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2875; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2876; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2877; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2878; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2879; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2880; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2881; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2882; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2883; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2884; GFX11-WGP-NEXT:    buffer_gl0_inv
2885; GFX11-WGP-NEXT:    s_endpgm
2886;
2887; GFX11-CU-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
2888; GFX11-CU:       ; %bb.0: ; %entry
2889; GFX11-CU-NEXT:    s_clause 0x1
2890; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2891; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2892; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2893; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2894; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2895; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2896; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2897; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2898; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2899; GFX11-CU-NEXT:    s_endpgm
2900    i32* %out, i32 %in, i32 %old) {
2901entry:
2902  %gep = getelementptr i32, i32* %out, i32 4
2903  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
2904  ret void
2905}
2906
2907define amdgpu_kernel void @flat_workgroup_monotonic_acquire_cmpxchg(
2908; GFX7-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
2909; GFX7:       ; %bb.0: ; %entry
2910; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2911; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2912; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2913; GFX7-NEXT:    s_add_u32 s0, s0, 16
2914; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2915; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2916; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2917; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2918; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2919; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2920; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2921; GFX7-NEXT:    s_endpgm
2922;
2923; GFX10-WGP-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
2924; GFX10-WGP:       ; %bb.0: ; %entry
2925; GFX10-WGP-NEXT:    s_clause 0x1
2926; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2927; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2928; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2929; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2930; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2931; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2932; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2933; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2934; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2935; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2936; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2937; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2938; GFX10-WGP-NEXT:    buffer_gl0_inv
2939; GFX10-WGP-NEXT:    s_endpgm
2940;
2941; GFX10-CU-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
2942; GFX10-CU:       ; %bb.0: ; %entry
2943; GFX10-CU-NEXT:    s_clause 0x1
2944; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2945; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2946; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2947; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2948; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2949; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2950; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2951; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2952; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2953; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2954; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2955; GFX10-CU-NEXT:    s_endpgm
2956;
2957; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
2958; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2959; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2960; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2961; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2962; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2963; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2964; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2965; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2966; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2967; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2968; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2969; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2970; SKIP-CACHE-INV-NEXT:    s_endpgm
2971;
2972; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
2973; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2974; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2975; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2976; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2977; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2978; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2979; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2980; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2981; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2982;
2983; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
2984; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2985; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2986; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2987; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2988; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2989; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2990; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2991; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2992; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2993; GFX90A-TGSPLIT-NEXT:    s_endpgm
2994;
2995; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
2996; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2997; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2998; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2999; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3000; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3001; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3002; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3003; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3004; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3005;
3006; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
3007; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3008; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3009; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3010; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3011; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3012; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3013; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3014; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3015; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
3016; GFX940-TGSPLIT-NEXT:    s_endpgm
3017;
3018; GFX11-WGP-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
3019; GFX11-WGP:       ; %bb.0: ; %entry
3020; GFX11-WGP-NEXT:    s_clause 0x1
3021; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3022; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3023; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3024; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3025; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3026; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3027; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3028; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3029; GFX11-WGP-NEXT:    buffer_gl0_inv
3030; GFX11-WGP-NEXT:    s_endpgm
3031;
3032; GFX11-CU-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
3033; GFX11-CU:       ; %bb.0: ; %entry
3034; GFX11-CU-NEXT:    s_clause 0x1
3035; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3036; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3037; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3038; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3039; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3040; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3041; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3042; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3043; GFX11-CU-NEXT:    s_endpgm
3044    i32* %out, i32 %in, i32 %old) {
3045entry:
3046  %gep = getelementptr i32, i32* %out, i32 4
3047  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic acquire
3048  ret void
3049}
3050
3051define amdgpu_kernel void @flat_workgroup_acquire_acquire_cmpxchg(
3052; GFX7-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
3053; GFX7:       ; %bb.0: ; %entry
3054; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3055; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3056; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3057; GFX7-NEXT:    s_add_u32 s0, s0, 16
3058; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3059; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3060; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3061; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3062; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3063; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3064; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3065; GFX7-NEXT:    s_endpgm
3066;
3067; GFX10-WGP-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
3068; GFX10-WGP:       ; %bb.0: ; %entry
3069; GFX10-WGP-NEXT:    s_clause 0x1
3070; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3071; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3072; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3073; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3074; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3075; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3076; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3077; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3078; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3079; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3080; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3081; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3082; GFX10-WGP-NEXT:    buffer_gl0_inv
3083; GFX10-WGP-NEXT:    s_endpgm
3084;
3085; GFX10-CU-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
3086; GFX10-CU:       ; %bb.0: ; %entry
3087; GFX10-CU-NEXT:    s_clause 0x1
3088; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3089; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3090; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3091; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3092; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3093; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3094; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3095; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3096; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3097; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3098; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3099; GFX10-CU-NEXT:    s_endpgm
3100;
3101; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
3102; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3103; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3104; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3105; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3106; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3107; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3108; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3109; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3110; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3111; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3112; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3113; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3114; SKIP-CACHE-INV-NEXT:    s_endpgm
3115;
3116; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
3117; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3118; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3119; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3120; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3121; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3122; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3123; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3124; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3125; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3126;
3127; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
3128; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3129; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3130; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3131; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3132; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3133; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3134; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3135; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3136; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3137; GFX90A-TGSPLIT-NEXT:    s_endpgm
3138;
3139; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
3140; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3141; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3142; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3143; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3144; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3145; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3146; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3147; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3148; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3149;
3150; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
3151; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3152; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3153; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3154; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3155; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3156; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3157; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3158; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3159; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
3160; GFX940-TGSPLIT-NEXT:    s_endpgm
3161;
3162; GFX11-WGP-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
3163; GFX11-WGP:       ; %bb.0: ; %entry
3164; GFX11-WGP-NEXT:    s_clause 0x1
3165; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3166; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3167; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3168; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3169; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3170; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3171; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3172; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3173; GFX11-WGP-NEXT:    buffer_gl0_inv
3174; GFX11-WGP-NEXT:    s_endpgm
3175;
3176; GFX11-CU-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
3177; GFX11-CU:       ; %bb.0: ; %entry
3178; GFX11-CU-NEXT:    s_clause 0x1
3179; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3180; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3181; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3182; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3183; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3184; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3185; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3186; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3187; GFX11-CU-NEXT:    s_endpgm
3188    i32* %out, i32 %in, i32 %old) {
3189entry:
3190  %gep = getelementptr i32, i32* %out, i32 4
3191  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
3192  ret void
3193}
3194
3195define amdgpu_kernel void @flat_workgroup_release_acquire_cmpxchg(
3196; GFX7-LABEL: flat_workgroup_release_acquire_cmpxchg:
3197; GFX7:       ; %bb.0: ; %entry
3198; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3199; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3200; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3201; GFX7-NEXT:    s_add_u32 s0, s0, 16
3202; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3203; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3204; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3205; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3206; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3207; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3208; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3209; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3210; GFX7-NEXT:    s_endpgm
3211;
3212; GFX10-WGP-LABEL: flat_workgroup_release_acquire_cmpxchg:
3213; GFX10-WGP:       ; %bb.0: ; %entry
3214; GFX10-WGP-NEXT:    s_clause 0x1
3215; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3216; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3217; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3218; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3219; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3220; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3221; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3222; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3223; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3224; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3225; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3226; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3227; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3228; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3229; GFX10-WGP-NEXT:    buffer_gl0_inv
3230; GFX10-WGP-NEXT:    s_endpgm
3231;
3232; GFX10-CU-LABEL: flat_workgroup_release_acquire_cmpxchg:
3233; GFX10-CU:       ; %bb.0: ; %entry
3234; GFX10-CU-NEXT:    s_clause 0x1
3235; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3236; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3237; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3238; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3239; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3240; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3241; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3242; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3243; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3244; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3245; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3246; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3247; GFX10-CU-NEXT:    s_endpgm
3248;
3249; SKIP-CACHE-INV-LABEL: flat_workgroup_release_acquire_cmpxchg:
3250; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3251; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3252; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3253; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3254; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3255; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3256; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3257; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3258; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3259; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3260; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3261; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3262; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3263; SKIP-CACHE-INV-NEXT:    s_endpgm
3264;
3265; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg:
3266; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3267; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3268; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3269; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3270; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3271; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3272; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3273; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3274; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3275; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3276;
3277; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg:
3278; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3279; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3280; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3281; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3282; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3283; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3284; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3285; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3286; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3287; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3288; GFX90A-TGSPLIT-NEXT:    s_endpgm
3289;
3290; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg:
3291; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3292; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3293; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3294; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3295; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3296; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3297; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3298; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3299; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3300; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3301;
3302; GFX940-TGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg:
3303; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3304; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3305; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3306; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3307; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3308; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3309; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3310; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3311; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3312; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
3313; GFX940-TGSPLIT-NEXT:    s_endpgm
3314;
3315; GFX11-WGP-LABEL: flat_workgroup_release_acquire_cmpxchg:
3316; GFX11-WGP:       ; %bb.0: ; %entry
3317; GFX11-WGP-NEXT:    s_clause 0x1
3318; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3319; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3320; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3321; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3322; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3323; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3324; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3325; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3326; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3327; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3328; GFX11-WGP-NEXT:    buffer_gl0_inv
3329; GFX11-WGP-NEXT:    s_endpgm
3330;
3331; GFX11-CU-LABEL: flat_workgroup_release_acquire_cmpxchg:
3332; GFX11-CU:       ; %bb.0: ; %entry
3333; GFX11-CU-NEXT:    s_clause 0x1
3334; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3335; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3336; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3337; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3338; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3339; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3340; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3341; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3342; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3343; GFX11-CU-NEXT:    s_endpgm
3344    i32* %out, i32 %in, i32 %old) {
3345entry:
3346  %gep = getelementptr i32, i32* %out, i32 4
3347  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
3348  ret void
3349}
3350
3351define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_cmpxchg(
3352; GFX7-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
3353; GFX7:       ; %bb.0: ; %entry
3354; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3355; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3356; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3357; GFX7-NEXT:    s_add_u32 s0, s0, 16
3358; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3359; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3360; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3361; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3362; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3363; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3364; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3365; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3366; GFX7-NEXT:    s_endpgm
3367;
3368; GFX10-WGP-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
3369; GFX10-WGP:       ; %bb.0: ; %entry
3370; GFX10-WGP-NEXT:    s_clause 0x1
3371; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3372; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3373; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3374; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3375; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3376; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3377; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3378; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3379; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3380; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3381; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3382; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3383; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3384; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3385; GFX10-WGP-NEXT:    buffer_gl0_inv
3386; GFX10-WGP-NEXT:    s_endpgm
3387;
3388; GFX10-CU-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
3389; GFX10-CU:       ; %bb.0: ; %entry
3390; GFX10-CU-NEXT:    s_clause 0x1
3391; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3392; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3393; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3394; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3395; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3396; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3397; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3398; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3399; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3400; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3401; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3402; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3403; GFX10-CU-NEXT:    s_endpgm
3404;
3405; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
3406; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3407; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3408; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3409; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3410; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3411; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3412; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3413; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3414; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3415; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3416; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3417; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3418; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3419; SKIP-CACHE-INV-NEXT:    s_endpgm
3420;
3421; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
3422; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3423; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3424; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3425; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3426; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3427; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3428; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3429; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3430; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3431; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3432;
3433; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
3434; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3435; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3436; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3437; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3438; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3439; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3440; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3441; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3442; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3443; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3444; GFX90A-TGSPLIT-NEXT:    s_endpgm
3445;
3446; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
3447; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3448; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3449; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3450; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3451; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3452; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3453; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3454; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3455; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3456; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3457;
3458; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
3459; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3460; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3461; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3462; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3463; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3464; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3465; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3466; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3467; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3468; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
3469; GFX940-TGSPLIT-NEXT:    s_endpgm
3470;
3471; GFX11-WGP-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
3472; GFX11-WGP:       ; %bb.0: ; %entry
3473; GFX11-WGP-NEXT:    s_clause 0x1
3474; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3475; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3476; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3477; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3478; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3479; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3480; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3481; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3482; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3483; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3484; GFX11-WGP-NEXT:    buffer_gl0_inv
3485; GFX11-WGP-NEXT:    s_endpgm
3486;
3487; GFX11-CU-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
3488; GFX11-CU:       ; %bb.0: ; %entry
3489; GFX11-CU-NEXT:    s_clause 0x1
3490; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3491; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3492; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3493; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3494; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3495; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3496; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3497; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3498; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3499; GFX11-CU-NEXT:    s_endpgm
3500    i32* %out, i32 %in, i32 %old) {
3501entry:
3502  %gep = getelementptr i32, i32* %out, i32 4
3503  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
3504  ret void
3505}
3506
3507define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_cmpxchg(
3508; GFX7-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
3509; GFX7:       ; %bb.0: ; %entry
3510; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3511; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3512; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3513; GFX7-NEXT:    s_add_u32 s0, s0, 16
3514; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3515; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3516; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3517; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3518; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3519; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3520; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3521; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3522; GFX7-NEXT:    s_endpgm
3523;
3524; GFX10-WGP-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
3525; GFX10-WGP:       ; %bb.0: ; %entry
3526; GFX10-WGP-NEXT:    s_clause 0x1
3527; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3528; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3529; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3530; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3531; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3532; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3533; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3534; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3535; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3536; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3537; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3538; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3539; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3540; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3541; GFX10-WGP-NEXT:    buffer_gl0_inv
3542; GFX10-WGP-NEXT:    s_endpgm
3543;
3544; GFX10-CU-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
3545; GFX10-CU:       ; %bb.0: ; %entry
3546; GFX10-CU-NEXT:    s_clause 0x1
3547; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3548; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3549; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3550; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3551; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3552; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3553; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3554; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3555; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3556; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3557; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3558; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3559; GFX10-CU-NEXT:    s_endpgm
3560;
3561; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
3562; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3563; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3564; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3565; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3566; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3567; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3568; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3569; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3570; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3571; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3572; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3573; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3574; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3575; SKIP-CACHE-INV-NEXT:    s_endpgm
3576;
3577; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
3578; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3579; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3580; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3581; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3582; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3583; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3584; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3585; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3586; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3587; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3588;
3589; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
3590; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3591; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3592; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3593; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3594; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3595; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3596; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3597; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3598; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3599; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3600; GFX90A-TGSPLIT-NEXT:    s_endpgm
3601;
3602; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
3603; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3604; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3605; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3606; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3607; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3608; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3609; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3610; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3611; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3612; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3613;
3614; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
3615; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3616; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3617; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3618; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3619; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3620; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3621; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3622; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3623; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3624; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
3625; GFX940-TGSPLIT-NEXT:    s_endpgm
3626;
3627; GFX11-WGP-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
3628; GFX11-WGP:       ; %bb.0: ; %entry
3629; GFX11-WGP-NEXT:    s_clause 0x1
3630; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3631; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3632; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3633; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3634; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3635; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3636; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3637; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3638; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3639; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3640; GFX11-WGP-NEXT:    buffer_gl0_inv
3641; GFX11-WGP-NEXT:    s_endpgm
3642;
3643; GFX11-CU-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
3644; GFX11-CU:       ; %bb.0: ; %entry
3645; GFX11-CU-NEXT:    s_clause 0x1
3646; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3647; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3648; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3649; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3650; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3651; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3652; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3653; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3654; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3655; GFX11-CU-NEXT:    s_endpgm
3656    i32* %out, i32 %in, i32 %old) {
3657entry:
3658  %gep = getelementptr i32, i32* %out, i32 4
3659  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
3660  ret void
3661}
3662
3663define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_cmpxchg(
3664; GFX7-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
3665; GFX7:       ; %bb.0: ; %entry
3666; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3667; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3668; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3669; GFX7-NEXT:    s_add_u32 s0, s0, 16
3670; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3671; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3672; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3673; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3674; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3675; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3676; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3677; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3678; GFX7-NEXT:    s_endpgm
3679;
3680; GFX10-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
3681; GFX10-WGP:       ; %bb.0: ; %entry
3682; GFX10-WGP-NEXT:    s_clause 0x1
3683; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3684; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3685; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3686; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3687; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3688; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3689; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3690; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3691; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3692; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3693; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3694; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3695; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3696; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3697; GFX10-WGP-NEXT:    buffer_gl0_inv
3698; GFX10-WGP-NEXT:    s_endpgm
3699;
3700; GFX10-CU-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
3701; GFX10-CU:       ; %bb.0: ; %entry
3702; GFX10-CU-NEXT:    s_clause 0x1
3703; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3704; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3705; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3706; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3707; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3708; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3709; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3710; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3711; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3712; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3713; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3714; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3715; GFX10-CU-NEXT:    s_endpgm
3716;
3717; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
3718; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3719; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3720; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3721; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3722; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3723; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3724; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3725; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3726; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3727; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3728; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3729; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3730; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3731; SKIP-CACHE-INV-NEXT:    s_endpgm
3732;
3733; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
3734; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3735; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3736; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3737; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3738; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3739; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3740; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3741; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3742; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3743; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3744;
3745; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
3746; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3747; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3748; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3749; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3750; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3751; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3752; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3753; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3754; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3755; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3756; GFX90A-TGSPLIT-NEXT:    s_endpgm
3757;
3758; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
3759; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3760; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3761; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3762; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3763; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3764; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3765; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3766; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3767; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3768; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3769;
3770; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
3771; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3772; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3773; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3774; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3775; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3776; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3777; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3778; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3779; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3780; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
3781; GFX940-TGSPLIT-NEXT:    s_endpgm
3782;
3783; GFX11-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
3784; GFX11-WGP:       ; %bb.0: ; %entry
3785; GFX11-WGP-NEXT:    s_clause 0x1
3786; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3787; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3788; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3789; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3790; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3791; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3792; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3793; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3794; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3795; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3796; GFX11-WGP-NEXT:    buffer_gl0_inv
3797; GFX11-WGP-NEXT:    s_endpgm
3798;
3799; GFX11-CU-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
3800; GFX11-CU:       ; %bb.0: ; %entry
3801; GFX11-CU-NEXT:    s_clause 0x1
3802; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3803; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3804; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3805; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3806; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3807; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3808; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3809; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3810; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3811; GFX11-CU-NEXT:    s_endpgm
3812    i32* %out, i32 %in, i32 %old) {
3813entry:
3814  %gep = getelementptr i32, i32* %out, i32 4
3815  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
3816  ret void
3817}
3818
3819define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_ret_cmpxchg(
3820; GFX7-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
3821; GFX7:       ; %bb.0: ; %entry
3822; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3823; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3824; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3825; GFX7-NEXT:    s_add_u32 s4, s0, 16
3826; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3827; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3828; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3829; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3830; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3831; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3832; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3833; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3834; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3835; GFX7-NEXT:    flat_store_dword v[0:1], v2
3836; GFX7-NEXT:    s_endpgm
3837;
3838; GFX10-WGP-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
3839; GFX10-WGP:       ; %bb.0: ; %entry
3840; GFX10-WGP-NEXT:    s_clause 0x1
3841; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3842; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3843; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3844; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3845; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3846; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3847; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3848; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3849; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3850; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3851; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3852; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3853; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3854; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3855; GFX10-WGP-NEXT:    s_endpgm
3856;
3857; GFX10-CU-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
3858; GFX10-CU:       ; %bb.0: ; %entry
3859; GFX10-CU-NEXT:    s_clause 0x1
3860; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3861; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3862; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3863; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3864; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3865; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3866; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3867; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3868; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3869; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3870; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3871; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3872; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3873; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3874; GFX10-CU-NEXT:    s_endpgm
3875;
3876; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
3877; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3878; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3879; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3880; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3881; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3882; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3883; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3884; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3885; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3886; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3887; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3888; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3889; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3890; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3891; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3892; SKIP-CACHE-INV-NEXT:    s_endpgm
3893;
3894; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
3895; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3896; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3897; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3898; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3899; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3900; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3901; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3902; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3903; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3904; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3905;
3906; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
3907; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3908; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3909; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3910; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3911; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3912; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3913; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3914; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3915; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3916; GFX90A-TGSPLIT-NEXT:    s_endpgm
3917;
3918; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
3919; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3920; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3921; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3922; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3923; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3924; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3925; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3926; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3927; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3928; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3929;
3930; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
3931; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3932; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3933; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3934; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3935; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3936; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3937; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3938; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3939; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3940; GFX940-TGSPLIT-NEXT:    s_endpgm
3941;
3942; GFX11-WGP-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
3943; GFX11-WGP:       ; %bb.0: ; %entry
3944; GFX11-WGP-NEXT:    s_clause 0x1
3945; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3946; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3947; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3948; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3949; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3950; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
3951; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3952; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
3953; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3954; GFX11-WGP-NEXT:    s_endpgm
3955;
3956; GFX11-CU-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
3957; GFX11-CU:       ; %bb.0: ; %entry
3958; GFX11-CU-NEXT:    s_clause 0x1
3959; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3960; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3961; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3962; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3963; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3964; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
3965; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3966; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
3967; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3968; GFX11-CU-NEXT:    s_endpgm
3969    i32* %out, i32 %in, i32 %old) {
3970entry:
3971  %gep = getelementptr i32, i32* %out, i32 4
3972  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic
3973  %val0 = extractvalue { i32, i1 } %val, 0
3974  store i32 %val0, i32* %out, align 4
3975  ret void
3976}
3977
3978define amdgpu_kernel void @flat_workgroup_acquire_monotonic_ret_cmpxchg(
3979; GFX7-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
3980; GFX7:       ; %bb.0: ; %entry
3981; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3982; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3983; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3984; GFX7-NEXT:    s_add_u32 s4, s0, 16
3985; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3986; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3987; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3988; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3989; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3990; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3991; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3992; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3993; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3994; GFX7-NEXT:    s_waitcnt vmcnt(0)
3995; GFX7-NEXT:    flat_store_dword v[0:1], v2
3996; GFX7-NEXT:    s_endpgm
3997;
3998; GFX10-WGP-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
3999; GFX10-WGP:       ; %bb.0: ; %entry
4000; GFX10-WGP-NEXT:    s_clause 0x1
4001; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4002; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4003; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4004; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4005; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4006; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4007; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4008; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4009; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4010; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4011; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4012; GFX10-WGP-NEXT:    buffer_gl0_inv
4013; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4014; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4015; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4016; GFX10-WGP-NEXT:    s_endpgm
4017;
4018; GFX10-CU-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
4019; GFX10-CU:       ; %bb.0: ; %entry
4020; GFX10-CU-NEXT:    s_clause 0x1
4021; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4022; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4023; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4024; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4025; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4026; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4027; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4028; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4029; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4030; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4031; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4032; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4033; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4034; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4035; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4036; GFX10-CU-NEXT:    s_endpgm
4037;
4038; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
4039; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4040; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4041; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4042; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4043; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4044; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4045; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4046; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4047; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4048; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4049; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4050; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4051; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4052; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4053; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4054; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4055; SKIP-CACHE-INV-NEXT:    s_endpgm
4056;
4057; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
4058; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4059; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4060; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4061; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4062; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4063; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4064; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4065; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4066; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4067; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4068;
4069; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
4070; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4071; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4072; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4073; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4074; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4075; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4076; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4077; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4078; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4079; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4080; GFX90A-TGSPLIT-NEXT:    s_endpgm
4081;
4082; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
4083; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4084; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4085; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4086; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4087; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4088; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4089; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4090; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4091; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4092; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4093;
4094; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
4095; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4096; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4097; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4098; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4099; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4100; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4101; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4102; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4103; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
4104; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4105; GFX940-TGSPLIT-NEXT:    s_endpgm
4106;
4107; GFX11-WGP-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
4108; GFX11-WGP:       ; %bb.0: ; %entry
4109; GFX11-WGP-NEXT:    s_clause 0x1
4110; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4111; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4112; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4113; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4114; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4115; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4116; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4117; GFX11-WGP-NEXT:    buffer_gl0_inv
4118; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4119; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4120; GFX11-WGP-NEXT:    s_endpgm
4121;
4122; GFX11-CU-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
4123; GFX11-CU:       ; %bb.0: ; %entry
4124; GFX11-CU-NEXT:    s_clause 0x1
4125; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4126; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4127; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4128; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4129; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4130; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4131; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4132; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4133; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4134; GFX11-CU-NEXT:    s_endpgm
4135    i32* %out, i32 %in, i32 %old) {
4136entry:
4137  %gep = getelementptr i32, i32* %out, i32 4
4138  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
4139  %val0 = extractvalue { i32, i1 } %val, 0
4140  store i32 %val0, i32* %out, align 4
4141  ret void
4142}
4143
4144define amdgpu_kernel void @flat_workgroup_release_monotonic_ret_cmpxchg(
4145; GFX7-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
4146; GFX7:       ; %bb.0: ; %entry
4147; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4148; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4149; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4150; GFX7-NEXT:    s_add_u32 s4, s0, 16
4151; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4152; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4153; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4154; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4155; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4156; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4157; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4158; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4159; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4160; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4161; GFX7-NEXT:    flat_store_dword v[0:1], v2
4162; GFX7-NEXT:    s_endpgm
4163;
4164; GFX10-WGP-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
4165; GFX10-WGP:       ; %bb.0: ; %entry
4166; GFX10-WGP-NEXT:    s_clause 0x1
4167; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4168; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4169; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4170; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4171; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4172; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4173; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4174; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4175; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4176; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4177; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4178; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4179; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4180; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4181; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4182; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4183; GFX10-WGP-NEXT:    s_endpgm
4184;
4185; GFX10-CU-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
4186; GFX10-CU:       ; %bb.0: ; %entry
4187; GFX10-CU-NEXT:    s_clause 0x1
4188; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4189; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4190; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4191; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4192; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4193; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4194; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4195; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4196; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4197; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4198; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4199; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4200; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4201; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4202; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4203; GFX10-CU-NEXT:    s_endpgm
4204;
4205; SKIP-CACHE-INV-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
4206; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4207; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4208; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4209; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4210; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4211; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4212; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4213; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4214; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4215; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4216; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4217; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4218; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4219; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4220; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4221; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4222; SKIP-CACHE-INV-NEXT:    s_endpgm
4223;
4224; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
4225; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4226; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4227; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4228; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4229; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4230; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4231; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4232; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4233; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4234; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4235; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4236;
4237; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
4238; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4239; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4240; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4241; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4242; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4243; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4244; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4245; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4246; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4247; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4248; GFX90A-TGSPLIT-NEXT:    s_endpgm
4249;
4250; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
4251; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4252; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4253; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4254; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4255; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4256; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4257; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4258; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4259; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4260; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4261; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4262;
4263; GFX940-TGSPLIT-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
4264; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4265; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4266; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4267; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4268; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4269; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4270; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4271; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4272; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4273; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4274; GFX940-TGSPLIT-NEXT:    s_endpgm
4275;
4276; GFX11-WGP-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
4277; GFX11-WGP:       ; %bb.0: ; %entry
4278; GFX11-WGP-NEXT:    s_clause 0x1
4279; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4280; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4281; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4282; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4283; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4284; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4285; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4286; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4287; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4288; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4289; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4290; GFX11-WGP-NEXT:    s_endpgm
4291;
4292; GFX11-CU-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
4293; GFX11-CU:       ; %bb.0: ; %entry
4294; GFX11-CU-NEXT:    s_clause 0x1
4295; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4296; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4297; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4298; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4299; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4300; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4301; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4302; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4303; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4304; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4305; GFX11-CU-NEXT:    s_endpgm
4306    i32* %out, i32 %in, i32 %old) {
4307entry:
4308  %gep = getelementptr i32, i32* %out, i32 4
4309  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic
4310  %val0 = extractvalue { i32, i1 } %val, 0
4311  store i32 %val0, i32* %out, align 4
4312  ret void
4313}
4314
4315define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_ret_cmpxchg(
4316; GFX7-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
4317; GFX7:       ; %bb.0: ; %entry
4318; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4319; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4320; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4321; GFX7-NEXT:    s_add_u32 s4, s0, 16
4322; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4323; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4324; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4325; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4326; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4327; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4328; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4329; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4330; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4331; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4332; GFX7-NEXT:    s_waitcnt vmcnt(0)
4333; GFX7-NEXT:    flat_store_dword v[0:1], v2
4334; GFX7-NEXT:    s_endpgm
4335;
4336; GFX10-WGP-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
4337; GFX10-WGP:       ; %bb.0: ; %entry
4338; GFX10-WGP-NEXT:    s_clause 0x1
4339; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4340; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4341; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4342; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4343; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4344; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4345; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4346; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4347; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4348; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4349; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4350; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4351; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4352; GFX10-WGP-NEXT:    buffer_gl0_inv
4353; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4354; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4355; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4356; GFX10-WGP-NEXT:    s_endpgm
4357;
4358; GFX10-CU-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
4359; GFX10-CU:       ; %bb.0: ; %entry
4360; GFX10-CU-NEXT:    s_clause 0x1
4361; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4362; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4363; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4364; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4365; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4366; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4367; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4368; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4369; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4370; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4371; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4372; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4373; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4374; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4375; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4376; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4377; GFX10-CU-NEXT:    s_endpgm
4378;
4379; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
4380; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4381; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4382; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4383; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4384; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4385; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4386; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4387; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4388; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4389; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4390; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4391; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4392; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4393; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4394; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4395; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4396; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4397; SKIP-CACHE-INV-NEXT:    s_endpgm
4398;
4399; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
4400; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4401; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4402; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4403; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4404; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4405; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4406; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4407; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4408; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4409; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4410; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4411;
4412; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
4413; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4414; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4415; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4416; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4417; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4418; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4419; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4420; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4421; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4422; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4423; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4424; GFX90A-TGSPLIT-NEXT:    s_endpgm
4425;
4426; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
4427; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4428; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4429; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4430; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4431; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4432; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4433; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4434; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4435; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4436; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4437; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4438;
4439; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
4440; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4441; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4442; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4443; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4444; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4445; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4446; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4447; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4448; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4449; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
4450; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4451; GFX940-TGSPLIT-NEXT:    s_endpgm
4452;
4453; GFX11-WGP-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
4454; GFX11-WGP:       ; %bb.0: ; %entry
4455; GFX11-WGP-NEXT:    s_clause 0x1
4456; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4457; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4458; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4459; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4460; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4461; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4462; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4463; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4464; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4465; GFX11-WGP-NEXT:    buffer_gl0_inv
4466; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4467; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4468; GFX11-WGP-NEXT:    s_endpgm
4469;
4470; GFX11-CU-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
4471; GFX11-CU:       ; %bb.0: ; %entry
4472; GFX11-CU-NEXT:    s_clause 0x1
4473; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4474; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4475; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4476; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4477; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4478; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4479; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4480; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4481; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4482; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4483; GFX11-CU-NEXT:    s_endpgm
4484    i32* %out, i32 %in, i32 %old) {
4485entry:
4486  %gep = getelementptr i32, i32* %out, i32 4
4487  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
4488  %val0 = extractvalue { i32, i1 } %val, 0
4489  store i32 %val0, i32* %out, align 4
4490  ret void
4491}
4492
4493define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_ret_cmpxchg(
4494; GFX7-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
4495; GFX7:       ; %bb.0: ; %entry
4496; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4497; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4498; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4499; GFX7-NEXT:    s_add_u32 s4, s0, 16
4500; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4501; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4502; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4503; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4504; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4505; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4506; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4507; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4508; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4509; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4510; GFX7-NEXT:    s_waitcnt vmcnt(0)
4511; GFX7-NEXT:    flat_store_dword v[0:1], v2
4512; GFX7-NEXT:    s_endpgm
4513;
4514; GFX10-WGP-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
4515; GFX10-WGP:       ; %bb.0: ; %entry
4516; GFX10-WGP-NEXT:    s_clause 0x1
4517; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4518; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4519; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4520; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4521; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4522; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4523; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4524; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4525; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4526; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4527; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4528; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4529; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4530; GFX10-WGP-NEXT:    buffer_gl0_inv
4531; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4532; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4533; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4534; GFX10-WGP-NEXT:    s_endpgm
4535;
4536; GFX10-CU-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
4537; GFX10-CU:       ; %bb.0: ; %entry
4538; GFX10-CU-NEXT:    s_clause 0x1
4539; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4540; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4541; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4542; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4543; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4544; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4545; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4546; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4547; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4548; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4549; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4550; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4551; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4552; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4553; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4554; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4555; GFX10-CU-NEXT:    s_endpgm
4556;
4557; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
4558; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4559; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4560; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4561; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4562; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4563; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4564; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4565; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4566; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4567; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4568; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4569; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4570; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4571; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4572; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4573; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4574; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4575; SKIP-CACHE-INV-NEXT:    s_endpgm
4576;
4577; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
4578; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4579; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4580; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4581; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4582; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4583; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4584; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4585; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4586; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4587; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4588; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4589;
4590; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
4591; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4592; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4593; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4594; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4595; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4596; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4597; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4598; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4599; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4600; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4601; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4602; GFX90A-TGSPLIT-NEXT:    s_endpgm
4603;
4604; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
4605; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4606; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4607; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4608; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4609; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4610; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4611; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4612; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4613; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4614; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4615; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4616;
4617; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
4618; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4619; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4620; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4621; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4622; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4623; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4624; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4625; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4626; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4627; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
4628; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4629; GFX940-TGSPLIT-NEXT:    s_endpgm
4630;
4631; GFX11-WGP-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
4632; GFX11-WGP:       ; %bb.0: ; %entry
4633; GFX11-WGP-NEXT:    s_clause 0x1
4634; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4635; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4636; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4637; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4638; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4639; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4640; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4641; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4642; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4643; GFX11-WGP-NEXT:    buffer_gl0_inv
4644; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4645; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4646; GFX11-WGP-NEXT:    s_endpgm
4647;
4648; GFX11-CU-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
4649; GFX11-CU:       ; %bb.0: ; %entry
4650; GFX11-CU-NEXT:    s_clause 0x1
4651; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4652; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4653; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4654; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4655; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4656; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4657; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4658; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4659; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4660; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4661; GFX11-CU-NEXT:    s_endpgm
4662    i32* %out, i32 %in, i32 %old) {
4663entry:
4664  %gep = getelementptr i32, i32* %out, i32 4
4665  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
4666  %val0 = extractvalue { i32, i1 } %val, 0
4667  store i32 %val0, i32* %out, align 4
4668  ret void
4669}
4670
4671define amdgpu_kernel void @flat_workgroup_monotonic_acquire_ret_cmpxchg(
4672; GFX7-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
4673; GFX7:       ; %bb.0: ; %entry
4674; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4675; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4676; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4677; GFX7-NEXT:    s_add_u32 s4, s0, 16
4678; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4679; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4680; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4681; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4682; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4683; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4684; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4685; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4686; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4687; GFX7-NEXT:    s_waitcnt vmcnt(0)
4688; GFX7-NEXT:    flat_store_dword v[0:1], v2
4689; GFX7-NEXT:    s_endpgm
4690;
4691; GFX10-WGP-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
4692; GFX10-WGP:       ; %bb.0: ; %entry
4693; GFX10-WGP-NEXT:    s_clause 0x1
4694; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4695; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4696; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4697; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4698; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4699; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4700; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4701; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4702; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4703; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4704; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4705; GFX10-WGP-NEXT:    buffer_gl0_inv
4706; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4707; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4708; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4709; GFX10-WGP-NEXT:    s_endpgm
4710;
4711; GFX10-CU-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
4712; GFX10-CU:       ; %bb.0: ; %entry
4713; GFX10-CU-NEXT:    s_clause 0x1
4714; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4715; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4716; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4717; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4718; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4719; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4720; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4721; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4722; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4723; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4724; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4725; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4726; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4727; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4728; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4729; GFX10-CU-NEXT:    s_endpgm
4730;
4731; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
4732; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4733; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4734; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4735; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4736; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4737; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4738; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4739; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4740; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4741; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4742; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4743; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4744; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4745; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4746; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4747; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4748; SKIP-CACHE-INV-NEXT:    s_endpgm
4749;
4750; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
4751; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4752; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4753; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4754; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4755; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4756; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4757; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4758; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4759; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4760; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4761;
4762; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
4763; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4764; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4765; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4766; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4767; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4768; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4769; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4770; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4771; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4772; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4773; GFX90A-TGSPLIT-NEXT:    s_endpgm
4774;
4775; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
4776; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4777; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4778; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4779; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4780; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4781; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4782; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4783; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4784; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4785; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4786;
4787; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
4788; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4789; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4790; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4791; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4792; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4793; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4794; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4795; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4796; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
4797; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4798; GFX940-TGSPLIT-NEXT:    s_endpgm
4799;
4800; GFX11-WGP-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
4801; GFX11-WGP:       ; %bb.0: ; %entry
4802; GFX11-WGP-NEXT:    s_clause 0x1
4803; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4804; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4805; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4806; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4807; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4808; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4809; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4810; GFX11-WGP-NEXT:    buffer_gl0_inv
4811; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4812; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4813; GFX11-WGP-NEXT:    s_endpgm
4814;
4815; GFX11-CU-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
4816; GFX11-CU:       ; %bb.0: ; %entry
4817; GFX11-CU-NEXT:    s_clause 0x1
4818; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4819; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4820; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4821; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4822; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4823; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4824; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4825; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4826; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4827; GFX11-CU-NEXT:    s_endpgm
4828    i32* %out, i32 %in, i32 %old) {
4829entry:
4830  %gep = getelementptr i32, i32* %out, i32 4
4831  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic acquire
4832  %val0 = extractvalue { i32, i1 } %val, 0
4833  store i32 %val0, i32* %out, align 4
4834  ret void
4835}
4836
4837define amdgpu_kernel void @flat_workgroup_acquire_acquire_ret_cmpxchg(
4838; GFX7-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
4839; GFX7:       ; %bb.0: ; %entry
4840; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4841; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4842; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4843; GFX7-NEXT:    s_add_u32 s4, s0, 16
4844; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4845; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4846; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4847; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4848; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4849; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4850; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4851; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4852; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4853; GFX7-NEXT:    s_waitcnt vmcnt(0)
4854; GFX7-NEXT:    flat_store_dword v[0:1], v2
4855; GFX7-NEXT:    s_endpgm
4856;
4857; GFX10-WGP-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
4858; GFX10-WGP:       ; %bb.0: ; %entry
4859; GFX10-WGP-NEXT:    s_clause 0x1
4860; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4861; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4862; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4863; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4864; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4865; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4866; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4867; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4868; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4869; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4870; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4871; GFX10-WGP-NEXT:    buffer_gl0_inv
4872; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4873; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4874; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4875; GFX10-WGP-NEXT:    s_endpgm
4876;
4877; GFX10-CU-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
4878; GFX10-CU:       ; %bb.0: ; %entry
4879; GFX10-CU-NEXT:    s_clause 0x1
4880; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4881; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4882; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4883; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4884; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4885; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4886; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4887; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4888; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4889; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4890; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4891; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4892; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4893; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4894; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4895; GFX10-CU-NEXT:    s_endpgm
4896;
4897; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
4898; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4899; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4900; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4901; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4902; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4903; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4904; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4905; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4906; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4907; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4908; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4909; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4910; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4911; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4912; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4913; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4914; SKIP-CACHE-INV-NEXT:    s_endpgm
4915;
4916; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
4917; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4918; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4919; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4920; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4921; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4922; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4923; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4924; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4925; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4926; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4927;
4928; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
4929; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4930; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4931; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4932; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4933; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4934; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4935; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4936; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4937; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4938; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4939; GFX90A-TGSPLIT-NEXT:    s_endpgm
4940;
4941; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
4942; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4943; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4944; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4945; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4946; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4947; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4948; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4949; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4950; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4951; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4952;
4953; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
4954; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4955; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4956; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4957; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4958; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4959; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4960; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4961; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4962; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
4963; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4964; GFX940-TGSPLIT-NEXT:    s_endpgm
4965;
4966; GFX11-WGP-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
4967; GFX11-WGP:       ; %bb.0: ; %entry
4968; GFX11-WGP-NEXT:    s_clause 0x1
4969; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4970; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4971; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4972; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4973; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4974; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4975; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4976; GFX11-WGP-NEXT:    buffer_gl0_inv
4977; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4978; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4979; GFX11-WGP-NEXT:    s_endpgm
4980;
4981; GFX11-CU-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
4982; GFX11-CU:       ; %bb.0: ; %entry
4983; GFX11-CU-NEXT:    s_clause 0x1
4984; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4985; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4986; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4987; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4988; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4989; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4990; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4991; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4992; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4993; GFX11-CU-NEXT:    s_endpgm
4994    i32* %out, i32 %in, i32 %old) {
4995entry:
4996  %gep = getelementptr i32, i32* %out, i32 4
4997  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
4998  %val0 = extractvalue { i32, i1 } %val, 0
4999  store i32 %val0, i32* %out, align 4
5000  ret void
5001}
5002
5003define amdgpu_kernel void @flat_workgroup_release_acquire_ret_cmpxchg(
5004; GFX7-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
5005; GFX7:       ; %bb.0: ; %entry
5006; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5007; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5008; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5009; GFX7-NEXT:    s_add_u32 s4, s0, 16
5010; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5011; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5012; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5013; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5014; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5015; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5016; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5017; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5018; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5019; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5020; GFX7-NEXT:    s_waitcnt vmcnt(0)
5021; GFX7-NEXT:    flat_store_dword v[0:1], v2
5022; GFX7-NEXT:    s_endpgm
5023;
5024; GFX10-WGP-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
5025; GFX10-WGP:       ; %bb.0: ; %entry
5026; GFX10-WGP-NEXT:    s_clause 0x1
5027; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5028; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5029; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5030; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5031; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5032; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5033; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5034; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5035; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5036; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5037; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5038; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5039; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5040; GFX10-WGP-NEXT:    buffer_gl0_inv
5041; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5042; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5043; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5044; GFX10-WGP-NEXT:    s_endpgm
5045;
5046; GFX10-CU-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
5047; GFX10-CU:       ; %bb.0: ; %entry
5048; GFX10-CU-NEXT:    s_clause 0x1
5049; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5050; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5051; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5052; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5053; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5054; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5055; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5056; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5057; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5058; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5059; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5060; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5061; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5062; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5063; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5064; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5065; GFX10-CU-NEXT:    s_endpgm
5066;
5067; SKIP-CACHE-INV-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
5068; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5069; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5070; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5071; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5072; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5073; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5074; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5075; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5076; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5077; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5078; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5079; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5080; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5081; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5082; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5083; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5084; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5085; SKIP-CACHE-INV-NEXT:    s_endpgm
5086;
5087; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
5088; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5089; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5090; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5091; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5092; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5093; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5094; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5095; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5096; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5097; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5098; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5099;
5100; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
5101; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5102; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5103; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5104; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5105; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5106; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5107; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5108; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5109; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5110; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5111; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5112; GFX90A-TGSPLIT-NEXT:    s_endpgm
5113;
5114; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
5115; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5116; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5117; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5118; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5119; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5120; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5121; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5122; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5123; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5124; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5125; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5126;
5127; GFX940-TGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
5128; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5129; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5130; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5131; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5132; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5133; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5134; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5135; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5136; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5137; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
5138; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5139; GFX940-TGSPLIT-NEXT:    s_endpgm
5140;
5141; GFX11-WGP-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
5142; GFX11-WGP:       ; %bb.0: ; %entry
5143; GFX11-WGP-NEXT:    s_clause 0x1
5144; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5145; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5146; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5147; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5148; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5149; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5150; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5151; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5152; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5153; GFX11-WGP-NEXT:    buffer_gl0_inv
5154; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5155; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5156; GFX11-WGP-NEXT:    s_endpgm
5157;
5158; GFX11-CU-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
5159; GFX11-CU:       ; %bb.0: ; %entry
5160; GFX11-CU-NEXT:    s_clause 0x1
5161; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5162; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5163; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5164; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5165; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5166; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5167; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5168; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5169; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5170; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5171; GFX11-CU-NEXT:    s_endpgm
5172    i32* %out, i32 %in, i32 %old) {
5173entry:
5174  %gep = getelementptr i32, i32* %out, i32 4
5175  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
5176  %val0 = extractvalue { i32, i1 } %val, 0
5177  store i32 %val0, i32* %out, align 4
5178  ret void
5179}
5180
5181define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_ret_cmpxchg(
5182; GFX7-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
5183; GFX7:       ; %bb.0: ; %entry
5184; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5185; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5186; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5187; GFX7-NEXT:    s_add_u32 s4, s0, 16
5188; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5189; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5190; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5191; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5192; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5193; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5194; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5195; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5196; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5197; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5198; GFX7-NEXT:    s_waitcnt vmcnt(0)
5199; GFX7-NEXT:    flat_store_dword v[0:1], v2
5200; GFX7-NEXT:    s_endpgm
5201;
5202; GFX10-WGP-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
5203; GFX10-WGP:       ; %bb.0: ; %entry
5204; GFX10-WGP-NEXT:    s_clause 0x1
5205; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5206; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5207; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5208; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5209; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5210; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5211; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5212; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5213; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5214; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5215; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5216; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5217; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5218; GFX10-WGP-NEXT:    buffer_gl0_inv
5219; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5220; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5221; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5222; GFX10-WGP-NEXT:    s_endpgm
5223;
5224; GFX10-CU-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
5225; GFX10-CU:       ; %bb.0: ; %entry
5226; GFX10-CU-NEXT:    s_clause 0x1
5227; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5228; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5229; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5230; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5231; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5232; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5233; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5234; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5235; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5236; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5237; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5238; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5239; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5240; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5241; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5242; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5243; GFX10-CU-NEXT:    s_endpgm
5244;
5245; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
5246; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5247; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5248; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5249; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5250; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5251; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5252; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5253; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5254; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5255; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5256; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5257; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5258; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5259; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5260; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5261; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5262; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5263; SKIP-CACHE-INV-NEXT:    s_endpgm
5264;
5265; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
5266; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5267; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5268; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5269; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5270; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5271; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5272; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5273; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5274; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5275; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5276; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5277;
5278; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
5279; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5280; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5281; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5282; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5283; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5284; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5285; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5286; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5287; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5288; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5289; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5290; GFX90A-TGSPLIT-NEXT:    s_endpgm
5291;
5292; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
5293; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5294; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5295; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5296; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5297; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5298; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5299; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5300; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5301; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5302; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5303; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5304;
5305; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
5306; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5307; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5308; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5309; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5310; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5311; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5312; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5313; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5314; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5315; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
5316; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5317; GFX940-TGSPLIT-NEXT:    s_endpgm
5318;
5319; GFX11-WGP-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
5320; GFX11-WGP:       ; %bb.0: ; %entry
5321; GFX11-WGP-NEXT:    s_clause 0x1
5322; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5323; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5324; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5325; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5326; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5327; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5328; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5329; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5330; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5331; GFX11-WGP-NEXT:    buffer_gl0_inv
5332; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5333; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5334; GFX11-WGP-NEXT:    s_endpgm
5335;
5336; GFX11-CU-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
5337; GFX11-CU:       ; %bb.0: ; %entry
5338; GFX11-CU-NEXT:    s_clause 0x1
5339; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5340; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5341; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5342; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5343; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5344; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5345; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5346; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5347; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5348; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5349; GFX11-CU-NEXT:    s_endpgm
5350    i32* %out, i32 %in, i32 %old) {
5351entry:
5352  %gep = getelementptr i32, i32* %out, i32 4
5353  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
5354  %val0 = extractvalue { i32, i1 } %val, 0
5355  store i32 %val0, i32* %out, align 4
5356  ret void
5357}
5358
5359define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_ret_cmpxchg(
5360; GFX7-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
5361; GFX7:       ; %bb.0: ; %entry
5362; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5363; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5364; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5365; GFX7-NEXT:    s_add_u32 s4, s0, 16
5366; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5367; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5368; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5369; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5370; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5371; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5372; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5373; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5374; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5375; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5376; GFX7-NEXT:    s_waitcnt vmcnt(0)
5377; GFX7-NEXT:    flat_store_dword v[0:1], v2
5378; GFX7-NEXT:    s_endpgm
5379;
5380; GFX10-WGP-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
5381; GFX10-WGP:       ; %bb.0: ; %entry
5382; GFX10-WGP-NEXT:    s_clause 0x1
5383; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5384; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5385; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5386; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5387; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5388; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5389; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5390; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5391; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5392; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5393; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5394; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5395; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5396; GFX10-WGP-NEXT:    buffer_gl0_inv
5397; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5398; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5399; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5400; GFX10-WGP-NEXT:    s_endpgm
5401;
5402; GFX10-CU-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
5403; GFX10-CU:       ; %bb.0: ; %entry
5404; GFX10-CU-NEXT:    s_clause 0x1
5405; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5406; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5407; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5408; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5409; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5410; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5411; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5412; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5413; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5414; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5415; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5416; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5417; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5418; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5419; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5420; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5421; GFX10-CU-NEXT:    s_endpgm
5422;
5423; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
5424; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5425; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5426; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5427; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5428; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5429; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5430; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5431; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5432; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5433; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5434; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5435; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5436; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5437; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5438; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5439; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5440; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5441; SKIP-CACHE-INV-NEXT:    s_endpgm
5442;
5443; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
5444; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5445; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5446; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5447; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5448; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5449; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5450; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5451; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5452; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5453; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5454; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5455;
5456; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
5457; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5458; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5459; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5460; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5461; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5462; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5463; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5464; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5465; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5466; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5467; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5468; GFX90A-TGSPLIT-NEXT:    s_endpgm
5469;
5470; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
5471; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5472; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5473; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5474; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5475; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5476; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5477; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5478; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5479; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5480; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5481; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5482;
5483; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
5484; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5485; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5486; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5487; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5488; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5489; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5490; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5491; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5492; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5493; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
5494; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5495; GFX940-TGSPLIT-NEXT:    s_endpgm
5496;
5497; GFX11-WGP-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
5498; GFX11-WGP:       ; %bb.0: ; %entry
5499; GFX11-WGP-NEXT:    s_clause 0x1
5500; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5501; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5502; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5503; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5504; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5505; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5506; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5507; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5508; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5509; GFX11-WGP-NEXT:    buffer_gl0_inv
5510; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5511; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5512; GFX11-WGP-NEXT:    s_endpgm
5513;
5514; GFX11-CU-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
5515; GFX11-CU:       ; %bb.0: ; %entry
5516; GFX11-CU-NEXT:    s_clause 0x1
5517; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5518; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5519; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5520; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5521; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5522; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5523; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5524; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5525; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5526; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5527; GFX11-CU-NEXT:    s_endpgm
5528    i32* %out, i32 %in, i32 %old) {
5529entry:
5530  %gep = getelementptr i32, i32* %out, i32 4
5531  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
5532  %val0 = extractvalue { i32, i1 } %val, 0
5533  store i32 %val0, i32* %out, align 4
5534  ret void
5535}
5536
5537define amdgpu_kernel void @flat_workgroup_monotonic_seq_cst_ret_cmpxchg(
5538; GFX7-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
5539; GFX7:       ; %bb.0: ; %entry
5540; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5541; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5542; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5543; GFX7-NEXT:    s_add_u32 s4, s0, 16
5544; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5545; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5546; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5547; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5548; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5549; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5550; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5551; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5552; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5553; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5554; GFX7-NEXT:    s_waitcnt vmcnt(0)
5555; GFX7-NEXT:    flat_store_dword v[0:1], v2
5556; GFX7-NEXT:    s_endpgm
5557;
5558; GFX10-WGP-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
5559; GFX10-WGP:       ; %bb.0: ; %entry
5560; GFX10-WGP-NEXT:    s_clause 0x1
5561; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5562; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5563; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5564; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5565; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5566; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5567; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5568; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5569; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5570; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5571; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5572; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5573; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5574; GFX10-WGP-NEXT:    buffer_gl0_inv
5575; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5576; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5577; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5578; GFX10-WGP-NEXT:    s_endpgm
5579;
5580; GFX10-CU-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
5581; GFX10-CU:       ; %bb.0: ; %entry
5582; GFX10-CU-NEXT:    s_clause 0x1
5583; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5584; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5585; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5586; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5587; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5588; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5589; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5590; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5591; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5592; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5593; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5594; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5595; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5596; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5597; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5598; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5599; GFX10-CU-NEXT:    s_endpgm
5600;
5601; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
5602; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5603; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5604; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5605; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5606; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5607; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5608; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5609; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5610; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5611; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5612; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5613; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5614; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5615; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5616; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5617; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5618; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5619; SKIP-CACHE-INV-NEXT:    s_endpgm
5620;
5621; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
5622; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5623; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5624; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5625; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5626; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5627; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5628; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5629; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5630; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5631; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5632; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5633;
5634; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
5635; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5636; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5637; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5638; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5639; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5640; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5641; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5642; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5643; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5644; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5645; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5646; GFX90A-TGSPLIT-NEXT:    s_endpgm
5647;
5648; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
5649; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5650; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5651; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5652; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5653; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5654; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5655; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5656; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5657; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5658; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5659; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5660;
5661; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
5662; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5663; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5664; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5665; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5666; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5667; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5668; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5669; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5670; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5671; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
5672; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5673; GFX940-TGSPLIT-NEXT:    s_endpgm
5674;
5675; GFX11-WGP-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
5676; GFX11-WGP:       ; %bb.0: ; %entry
5677; GFX11-WGP-NEXT:    s_clause 0x1
5678; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5679; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5680; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5681; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5682; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5683; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5684; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5685; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5686; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5687; GFX11-WGP-NEXT:    buffer_gl0_inv
5688; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5689; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5690; GFX11-WGP-NEXT:    s_endpgm
5691;
5692; GFX11-CU-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
5693; GFX11-CU:       ; %bb.0: ; %entry
5694; GFX11-CU-NEXT:    s_clause 0x1
5695; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5696; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5697; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5698; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5699; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5700; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5701; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5702; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5703; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5704; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5705; GFX11-CU-NEXT:    s_endpgm
5706    i32* %out, i32 %in, i32 %old) {
5707entry:
5708  %gep = getelementptr i32, i32* %out, i32 4
5709  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic seq_cst
5710  %val0 = extractvalue { i32, i1 } %val, 0
5711  store i32 %val0, i32* %out, align 4
5712  ret void
5713}
5714
5715define amdgpu_kernel void @flat_workgroup_acquire_seq_cst_ret_cmpxchg(
5716; GFX7-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
5717; GFX7:       ; %bb.0: ; %entry
5718; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5719; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5720; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5721; GFX7-NEXT:    s_add_u32 s4, s0, 16
5722; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5723; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5724; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5725; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5726; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5727; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5728; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5729; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5730; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5731; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5732; GFX7-NEXT:    s_waitcnt vmcnt(0)
5733; GFX7-NEXT:    flat_store_dword v[0:1], v2
5734; GFX7-NEXT:    s_endpgm
5735;
5736; GFX10-WGP-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
5737; GFX10-WGP:       ; %bb.0: ; %entry
5738; GFX10-WGP-NEXT:    s_clause 0x1
5739; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5740; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5741; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5742; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5743; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5744; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5745; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5746; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5747; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5748; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5749; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5750; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5751; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5752; GFX10-WGP-NEXT:    buffer_gl0_inv
5753; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5754; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5755; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5756; GFX10-WGP-NEXT:    s_endpgm
5757;
5758; GFX10-CU-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
5759; GFX10-CU:       ; %bb.0: ; %entry
5760; GFX10-CU-NEXT:    s_clause 0x1
5761; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5762; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5763; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5764; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5765; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5766; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5767; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5768; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5769; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5770; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5771; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5772; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5773; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5774; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5775; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5776; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5777; GFX10-CU-NEXT:    s_endpgm
5778;
5779; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
5780; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5781; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5782; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5783; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5784; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5785; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5786; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5787; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5788; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5789; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5790; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5791; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5792; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5793; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5794; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5795; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5796; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5797; SKIP-CACHE-INV-NEXT:    s_endpgm
5798;
5799; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
5800; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5801; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5802; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5803; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5804; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5805; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5806; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5807; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5808; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5809; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5810; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5811;
5812; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
5813; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5814; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5815; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5816; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5817; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5818; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5819; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5820; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5821; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5822; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5823; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5824; GFX90A-TGSPLIT-NEXT:    s_endpgm
5825;
5826; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
5827; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5828; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5829; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5830; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5831; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5832; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5833; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5834; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5835; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5836; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5837; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5838;
5839; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
5840; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5841; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5842; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5843; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5844; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5845; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5846; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5847; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5848; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5849; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
5850; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5851; GFX940-TGSPLIT-NEXT:    s_endpgm
5852;
5853; GFX11-WGP-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
5854; GFX11-WGP:       ; %bb.0: ; %entry
5855; GFX11-WGP-NEXT:    s_clause 0x1
5856; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5857; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5858; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5859; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5860; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5861; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5862; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5863; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5864; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5865; GFX11-WGP-NEXT:    buffer_gl0_inv
5866; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5867; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5868; GFX11-WGP-NEXT:    s_endpgm
5869;
5870; GFX11-CU-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
5871; GFX11-CU:       ; %bb.0: ; %entry
5872; GFX11-CU-NEXT:    s_clause 0x1
5873; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5874; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5875; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5876; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5877; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5878; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5879; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5880; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5881; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5882; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5883; GFX11-CU-NEXT:    s_endpgm
5884    i32* %out, i32 %in, i32 %old) {
5885entry:
5886  %gep = getelementptr i32, i32* %out, i32 4
5887  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire seq_cst
5888  %val0 = extractvalue { i32, i1 } %val, 0
5889  store i32 %val0, i32* %out, align 4
5890  ret void
5891}
5892
5893define amdgpu_kernel void @flat_workgroup_release_seq_cst_ret_cmpxchg(
5894; GFX7-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
5895; GFX7:       ; %bb.0: ; %entry
5896; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5897; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5898; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5899; GFX7-NEXT:    s_add_u32 s4, s0, 16
5900; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5901; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5902; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5903; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5904; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5905; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5906; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5907; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5908; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5909; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5910; GFX7-NEXT:    s_waitcnt vmcnt(0)
5911; GFX7-NEXT:    flat_store_dword v[0:1], v2
5912; GFX7-NEXT:    s_endpgm
5913;
5914; GFX10-WGP-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
5915; GFX10-WGP:       ; %bb.0: ; %entry
5916; GFX10-WGP-NEXT:    s_clause 0x1
5917; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5918; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5919; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5920; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5921; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5922; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5923; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5924; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5925; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5926; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5927; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5928; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5929; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5930; GFX10-WGP-NEXT:    buffer_gl0_inv
5931; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5932; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5933; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5934; GFX10-WGP-NEXT:    s_endpgm
5935;
5936; GFX10-CU-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
5937; GFX10-CU:       ; %bb.0: ; %entry
5938; GFX10-CU-NEXT:    s_clause 0x1
5939; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5940; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5941; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5942; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5943; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5944; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5945; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5946; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5947; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5948; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5949; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5950; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5951; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5952; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5953; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5954; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5955; GFX10-CU-NEXT:    s_endpgm
5956;
5957; SKIP-CACHE-INV-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
5958; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5959; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5960; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5961; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5962; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5963; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5964; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5965; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5966; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5967; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5968; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5969; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5970; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5971; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5972; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5973; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5974; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5975; SKIP-CACHE-INV-NEXT:    s_endpgm
5976;
5977; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
5978; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5979; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5980; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5981; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5982; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5983; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5984; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5985; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5986; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5987; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5988; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5989;
5990; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
5991; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5992; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5993; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5994; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5995; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5996; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5997; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5998; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5999; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6000; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6001; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6002; GFX90A-TGSPLIT-NEXT:    s_endpgm
6003;
6004; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
6005; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6006; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6007; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6008; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6009; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6010; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6011; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6012; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6013; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6014; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6015; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6016;
6017; GFX940-TGSPLIT-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
6018; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6019; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6020; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6021; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6022; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6023; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6024; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6025; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6026; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6027; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
6028; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6029; GFX940-TGSPLIT-NEXT:    s_endpgm
6030;
6031; GFX11-WGP-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
6032; GFX11-WGP:       ; %bb.0: ; %entry
6033; GFX11-WGP-NEXT:    s_clause 0x1
6034; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6035; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6036; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6037; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6038; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6039; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6040; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6041; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6042; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6043; GFX11-WGP-NEXT:    buffer_gl0_inv
6044; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6045; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6046; GFX11-WGP-NEXT:    s_endpgm
6047;
6048; GFX11-CU-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
6049; GFX11-CU:       ; %bb.0: ; %entry
6050; GFX11-CU-NEXT:    s_clause 0x1
6051; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6052; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6053; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6054; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6055; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6056; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6057; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6058; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6059; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6060; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6061; GFX11-CU-NEXT:    s_endpgm
6062    i32* %out, i32 %in, i32 %old) {
6063entry:
6064  %gep = getelementptr i32, i32* %out, i32 4
6065  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release seq_cst
6066  %val0 = extractvalue { i32, i1 } %val, 0
6067  store i32 %val0, i32* %out, align 4
6068  ret void
6069}
6070
6071define amdgpu_kernel void @flat_workgroup_acq_rel_seq_cst_ret_cmpxchg(
6072; GFX7-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
6073; GFX7:       ; %bb.0: ; %entry
6074; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6075; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6076; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6077; GFX7-NEXT:    s_add_u32 s4, s0, 16
6078; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6079; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6080; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6081; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6082; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6083; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6084; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6085; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6086; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6087; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6088; GFX7-NEXT:    s_waitcnt vmcnt(0)
6089; GFX7-NEXT:    flat_store_dword v[0:1], v2
6090; GFX7-NEXT:    s_endpgm
6091;
6092; GFX10-WGP-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
6093; GFX10-WGP:       ; %bb.0: ; %entry
6094; GFX10-WGP-NEXT:    s_clause 0x1
6095; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6096; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6097; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6098; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6099; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6100; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6101; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6102; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6103; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6104; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6105; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6106; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6107; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6108; GFX10-WGP-NEXT:    buffer_gl0_inv
6109; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6110; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6111; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6112; GFX10-WGP-NEXT:    s_endpgm
6113;
6114; GFX10-CU-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
6115; GFX10-CU:       ; %bb.0: ; %entry
6116; GFX10-CU-NEXT:    s_clause 0x1
6117; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6118; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6119; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6120; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6121; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6122; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6123; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6124; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6125; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6126; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6127; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6128; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6129; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6130; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6131; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6132; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6133; GFX10-CU-NEXT:    s_endpgm
6134;
6135; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
6136; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6137; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6138; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6139; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6140; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6141; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6142; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6143; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6144; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6145; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6146; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6147; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6148; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6149; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6150; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6151; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6152; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6153; SKIP-CACHE-INV-NEXT:    s_endpgm
6154;
6155; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
6156; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6157; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6158; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6159; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6160; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6161; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6162; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6163; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6164; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6165; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6166; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6167;
6168; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
6169; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6170; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6171; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6172; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6173; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6174; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6175; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6176; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6177; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6178; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6179; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6180; GFX90A-TGSPLIT-NEXT:    s_endpgm
6181;
6182; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
6183; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6184; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6185; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6186; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6187; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6188; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6189; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6190; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6191; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6192; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6193; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6194;
6195; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
6196; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6197; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6198; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6199; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6200; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6201; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6202; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6203; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6204; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6205; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
6206; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6207; GFX940-TGSPLIT-NEXT:    s_endpgm
6208;
6209; GFX11-WGP-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
6210; GFX11-WGP:       ; %bb.0: ; %entry
6211; GFX11-WGP-NEXT:    s_clause 0x1
6212; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6213; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6214; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6215; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6216; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6217; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6218; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6219; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6220; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6221; GFX11-WGP-NEXT:    buffer_gl0_inv
6222; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6223; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6224; GFX11-WGP-NEXT:    s_endpgm
6225;
6226; GFX11-CU-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
6227; GFX11-CU:       ; %bb.0: ; %entry
6228; GFX11-CU-NEXT:    s_clause 0x1
6229; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6230; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6231; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6232; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6233; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6234; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6235; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6236; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6237; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6238; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6239; GFX11-CU-NEXT:    s_endpgm
6240    i32* %out, i32 %in, i32 %old) {
6241entry:
6242  %gep = getelementptr i32, i32* %out, i32 4
6243  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel seq_cst
6244  %val0 = extractvalue { i32, i1 } %val, 0
6245  store i32 %val0, i32* %out, align 4
6246  ret void
6247}
6248
6249define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_ret_cmpxchg(
6250; GFX7-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
6251; GFX7:       ; %bb.0: ; %entry
6252; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6253; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6254; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6255; GFX7-NEXT:    s_add_u32 s4, s0, 16
6256; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6257; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6258; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6259; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6260; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6261; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6262; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6263; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6264; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6265; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6266; GFX7-NEXT:    s_waitcnt vmcnt(0)
6267; GFX7-NEXT:    flat_store_dword v[0:1], v2
6268; GFX7-NEXT:    s_endpgm
6269;
6270; GFX10-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
6271; GFX10-WGP:       ; %bb.0: ; %entry
6272; GFX10-WGP-NEXT:    s_clause 0x1
6273; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6274; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6275; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6276; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6277; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6278; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6279; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6280; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6281; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6282; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6283; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6284; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6285; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6286; GFX10-WGP-NEXT:    buffer_gl0_inv
6287; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6288; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6289; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6290; GFX10-WGP-NEXT:    s_endpgm
6291;
6292; GFX10-CU-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
6293; GFX10-CU:       ; %bb.0: ; %entry
6294; GFX10-CU-NEXT:    s_clause 0x1
6295; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6296; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6297; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6298; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6299; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6300; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6301; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6302; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6303; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6304; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6305; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6306; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6307; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6308; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6309; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6310; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6311; GFX10-CU-NEXT:    s_endpgm
6312;
6313; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
6314; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6315; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6316; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6317; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6318; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6319; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6320; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6321; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6322; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6323; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6324; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6325; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6326; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6327; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6328; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6329; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6330; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6331; SKIP-CACHE-INV-NEXT:    s_endpgm
6332;
6333; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
6334; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6335; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6336; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6337; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6338; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6339; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6340; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6341; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6342; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6343; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6344; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6345;
6346; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
6347; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6348; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6349; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6350; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6351; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6352; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6353; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6354; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6355; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6356; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6357; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6358; GFX90A-TGSPLIT-NEXT:    s_endpgm
6359;
6360; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
6361; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6362; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6363; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6364; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6365; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6366; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6367; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6368; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6369; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6370; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6371; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6372;
6373; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
6374; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6375; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6376; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6377; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6378; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6379; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6380; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6381; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6382; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6383; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
6384; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6385; GFX940-TGSPLIT-NEXT:    s_endpgm
6386;
6387; GFX11-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
6388; GFX11-WGP:       ; %bb.0: ; %entry
6389; GFX11-WGP-NEXT:    s_clause 0x1
6390; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6391; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6392; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6393; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6394; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6395; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6396; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6397; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6398; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6399; GFX11-WGP-NEXT:    buffer_gl0_inv
6400; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6401; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6402; GFX11-WGP-NEXT:    s_endpgm
6403;
6404; GFX11-CU-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
6405; GFX11-CU:       ; %bb.0: ; %entry
6406; GFX11-CU-NEXT:    s_clause 0x1
6407; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6408; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6409; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6410; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6411; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6412; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6413; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6414; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6415; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6416; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6417; GFX11-CU-NEXT:    s_endpgm
6418    i32* %out, i32 %in, i32 %old) {
6419entry:
6420  %gep = getelementptr i32, i32* %out, i32 4
6421  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
6422  %val0 = extractvalue { i32, i1 } %val, 0
6423  store i32 %val0, i32* %out, align 4
6424  ret void
6425}
6426
6427define amdgpu_kernel void @flat_workgroup_one_as_unordered_load(
6428; GFX7-LABEL: flat_workgroup_one_as_unordered_load:
6429; GFX7:       ; %bb.0: ; %entry
6430; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6431; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6432; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6433; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6434; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6435; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6436; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6437; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6438; GFX7-NEXT:    flat_store_dword v[0:1], v2
6439; GFX7-NEXT:    s_endpgm
6440;
6441; GFX10-WGP-LABEL: flat_workgroup_one_as_unordered_load:
6442; GFX10-WGP:       ; %bb.0: ; %entry
6443; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6444; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6445; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6446; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6447; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
6448; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6449; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6450; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6451; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6452; GFX10-WGP-NEXT:    s_endpgm
6453;
6454; GFX10-CU-LABEL: flat_workgroup_one_as_unordered_load:
6455; GFX10-CU:       ; %bb.0: ; %entry
6456; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6457; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6458; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6459; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6460; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6461; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6462; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6463; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6464; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6465; GFX10-CU-NEXT:    s_endpgm
6466;
6467; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_unordered_load:
6468; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6469; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6470; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6471; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6472; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6473; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
6474; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6475; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6476; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6477; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6478; SKIP-CACHE-INV-NEXT:    s_endpgm
6479;
6480; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_load:
6481; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6482; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6483; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6484; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6485; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6486; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6487; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6488; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6489; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6490; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6491; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6492;
6493; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_load:
6494; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6495; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6496; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6497; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6498; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6499; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6500; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6501; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6502; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6503; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6504; GFX90A-TGSPLIT-NEXT:    s_endpgm
6505;
6506; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_load:
6507; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6508; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6509; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6510; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6511; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6512; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6513; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6514; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6515; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6516; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6517; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6518;
6519; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_load:
6520; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6521; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6522; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6523; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6524; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6525; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6526; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6527; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6528; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6529; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6530; GFX940-TGSPLIT-NEXT:    s_endpgm
6531;
6532; GFX11-WGP-LABEL: flat_workgroup_one_as_unordered_load:
6533; GFX11-WGP:       ; %bb.0: ; %entry
6534; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6535; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6536; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6537; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
6538; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6539; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6540; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6541; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6542; GFX11-WGP-NEXT:    s_endpgm
6543;
6544; GFX11-CU-LABEL: flat_workgroup_one_as_unordered_load:
6545; GFX11-CU:       ; %bb.0: ; %entry
6546; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6547; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6548; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6549; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
6550; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6551; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6552; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6553; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6554; GFX11-CU-NEXT:    s_endpgm
6555    i32* %in, i32* %out) {
6556entry:
6557  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") unordered, align 4
6558  store i32 %val, i32* %out
6559  ret void
6560}
6561
6562define amdgpu_kernel void @flat_workgroup_one_as_monotonic_load(
6563; GFX7-LABEL: flat_workgroup_one_as_monotonic_load:
6564; GFX7:       ; %bb.0: ; %entry
6565; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6566; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6567; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6568; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6569; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6570; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6571; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6572; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6573; GFX7-NEXT:    flat_store_dword v[0:1], v2
6574; GFX7-NEXT:    s_endpgm
6575;
6576; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_load:
6577; GFX10-WGP:       ; %bb.0: ; %entry
6578; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6579; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6580; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6581; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6582; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
6583; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6584; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6585; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6586; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6587; GFX10-WGP-NEXT:    s_endpgm
6588;
6589; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_load:
6590; GFX10-CU:       ; %bb.0: ; %entry
6591; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6592; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6593; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6594; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6595; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6596; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6597; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6598; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6599; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6600; GFX10-CU-NEXT:    s_endpgm
6601;
6602; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_load:
6603; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6604; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6605; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6606; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6607; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6608; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
6609; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6610; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6611; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6612; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6613; SKIP-CACHE-INV-NEXT:    s_endpgm
6614;
6615; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load:
6616; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6617; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6618; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6619; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6620; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6621; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6622; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6623; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6624; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6625; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6626; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6627;
6628; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load:
6629; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6630; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6631; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6632; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6633; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6634; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
6635; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6636; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6637; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6638; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6639; GFX90A-TGSPLIT-NEXT:    s_endpgm
6640;
6641; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load:
6642; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6643; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6644; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6645; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6646; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6647; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
6648; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6649; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6650; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6651; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6652; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6653;
6654; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load:
6655; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6656; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6657; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6658; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6659; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6660; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
6661; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6662; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6663; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6664; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6665; GFX940-TGSPLIT-NEXT:    s_endpgm
6666;
6667; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_load:
6668; GFX11-WGP:       ; %bb.0: ; %entry
6669; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6670; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6671; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6672; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
6673; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6674; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6675; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6676; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6677; GFX11-WGP-NEXT:    s_endpgm
6678;
6679; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_load:
6680; GFX11-CU:       ; %bb.0: ; %entry
6681; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6682; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6683; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6684; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
6685; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6686; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6687; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6688; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6689; GFX11-CU-NEXT:    s_endpgm
6690    i32* %in, i32* %out) {
6691entry:
6692  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") monotonic, align 4
6693  store i32 %val, i32* %out
6694  ret void
6695}
6696
6697define amdgpu_kernel void @flat_workgroup_one_as_acquire_load(
6698; GFX7-LABEL: flat_workgroup_one_as_acquire_load:
6699; GFX7:       ; %bb.0: ; %entry
6700; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6701; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6702; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6703; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6704; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6705; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6706; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6707; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6708; GFX7-NEXT:    flat_store_dword v[0:1], v2
6709; GFX7-NEXT:    s_endpgm
6710;
6711; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_load:
6712; GFX10-WGP:       ; %bb.0: ; %entry
6713; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6714; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6715; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6716; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6717; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
6718; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6719; GFX10-WGP-NEXT:    buffer_gl0_inv
6720; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6721; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6722; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6723; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6724; GFX10-WGP-NEXT:    s_endpgm
6725;
6726; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_load:
6727; GFX10-CU:       ; %bb.0: ; %entry
6728; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6729; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6730; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6731; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6732; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6733; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6734; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6735; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6736; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6737; GFX10-CU-NEXT:    s_endpgm
6738;
6739; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_load:
6740; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6741; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6742; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6743; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6744; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6745; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
6746; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6747; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6748; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6749; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6750; SKIP-CACHE-INV-NEXT:    s_endpgm
6751;
6752; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_load:
6753; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6754; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6755; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6756; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6757; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6758; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6759; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6760; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6761; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6762; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6763; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6764;
6765; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_load:
6766; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6767; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6768; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6769; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6770; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6771; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
6772; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6773; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6774; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6775; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6776; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6777; GFX90A-TGSPLIT-NEXT:    s_endpgm
6778;
6779; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_load:
6780; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6781; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6782; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6783; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6784; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6785; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
6786; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6787; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6788; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6789; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6790; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6791;
6792; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_load:
6793; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6794; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6795; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6796; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6797; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6798; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
6799; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6800; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
6801; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6802; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6803; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6804; GFX940-TGSPLIT-NEXT:    s_endpgm
6805;
6806; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_load:
6807; GFX11-WGP:       ; %bb.0: ; %entry
6808; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6809; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6810; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6811; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
6812; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
6813; GFX11-WGP-NEXT:    buffer_gl0_inv
6814; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6815; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6816; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6817; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6818; GFX11-WGP-NEXT:    s_endpgm
6819;
6820; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_load:
6821; GFX11-CU:       ; %bb.0: ; %entry
6822; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6823; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6824; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6825; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
6826; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6827; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6828; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6829; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6830; GFX11-CU-NEXT:    s_endpgm
6831    i32* %in, i32* %out) {
6832entry:
6833  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") acquire, align 4
6834  store i32 %val, i32* %out
6835  ret void
6836}
6837
6838define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_load(
6839; GFX7-LABEL: flat_workgroup_one_as_seq_cst_load:
6840; GFX7:       ; %bb.0: ; %entry
6841; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6842; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6843; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6844; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6845; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6846; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6847; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6848; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6849; GFX7-NEXT:    flat_store_dword v[0:1], v2
6850; GFX7-NEXT:    s_endpgm
6851;
6852; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_load:
6853; GFX10-WGP:       ; %bb.0: ; %entry
6854; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6855; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6856; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6857; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6858; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6859; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6860; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
6861; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6862; GFX10-WGP-NEXT:    buffer_gl0_inv
6863; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6864; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6865; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6866; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6867; GFX10-WGP-NEXT:    s_endpgm
6868;
6869; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_load:
6870; GFX10-CU:       ; %bb.0: ; %entry
6871; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6872; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6873; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6874; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6875; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6876; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6877; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6878; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6879; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6880; GFX10-CU-NEXT:    s_endpgm
6881;
6882; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_load:
6883; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6884; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6885; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6886; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6887; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6888; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
6889; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6890; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6891; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6892; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6893; SKIP-CACHE-INV-NEXT:    s_endpgm
6894;
6895; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load:
6896; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6897; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6898; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6899; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6900; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6901; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6902; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6903; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6904; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6905; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6906; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6907;
6908; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load:
6909; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6910; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6911; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6912; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6913; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6914; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6915; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
6916; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6917; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6918; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6919; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6920; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6921; GFX90A-TGSPLIT-NEXT:    s_endpgm
6922;
6923; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load:
6924; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6925; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6926; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6927; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6928; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6929; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
6930; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6931; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6932; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6933; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6934; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6935;
6936; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load:
6937; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6938; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6939; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6940; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6941; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6942; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6943; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
6944; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6945; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
6946; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6947; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6948; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6949; GFX940-TGSPLIT-NEXT:    s_endpgm
6950;
6951; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_load:
6952; GFX11-WGP:       ; %bb.0: ; %entry
6953; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6954; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6955; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6956; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
6957; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6958; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
6959; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
6960; GFX11-WGP-NEXT:    buffer_gl0_inv
6961; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6962; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6963; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6964; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6965; GFX11-WGP-NEXT:    s_endpgm
6966;
6967; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_load:
6968; GFX11-CU:       ; %bb.0: ; %entry
6969; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6970; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6971; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6972; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
6973; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6974; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6975; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6976; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6977; GFX11-CU-NEXT:    s_endpgm
6978    i32* %in, i32* %out) {
6979entry:
6980  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") seq_cst, align 4
6981  store i32 %val, i32* %out
6982  ret void
6983}
6984
6985define amdgpu_kernel void @flat_workgroup_one_as_unordered_store(
6986; GFX7-LABEL: flat_workgroup_one_as_unordered_store:
6987; GFX7:       ; %bb.0: ; %entry
6988; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
6989; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
6990; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6991; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6992; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6993; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6994; GFX7-NEXT:    flat_store_dword v[0:1], v2
6995; GFX7-NEXT:    s_endpgm
6996;
6997; GFX10-WGP-LABEL: flat_workgroup_one_as_unordered_store:
6998; GFX10-WGP:       ; %bb.0: ; %entry
6999; GFX10-WGP-NEXT:    s_clause 0x1
7000; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7001; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7002; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7003; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7004; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7005; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7006; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7007; GFX10-WGP-NEXT:    s_endpgm
7008;
7009; GFX10-CU-LABEL: flat_workgroup_one_as_unordered_store:
7010; GFX10-CU:       ; %bb.0: ; %entry
7011; GFX10-CU-NEXT:    s_clause 0x1
7012; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7013; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7014; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7015; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7016; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7017; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7018; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7019; GFX10-CU-NEXT:    s_endpgm
7020;
7021; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_unordered_store:
7022; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7023; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
7024; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
7025; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7026; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7027; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7028; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7029; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7030; SKIP-CACHE-INV-NEXT:    s_endpgm
7031;
7032; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_store:
7033; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7034; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7035; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7036; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7037; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7038; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7039; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7040; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7041;
7042; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_store:
7043; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7044; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7045; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7046; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7047; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7048; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7049; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7050; GFX90A-TGSPLIT-NEXT:    s_endpgm
7051;
7052; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_store:
7053; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7054; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7055; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7056; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7057; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7058; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7059; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7060; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7061;
7062; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_store:
7063; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7064; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7065; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7066; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7067; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7068; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7069; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7070; GFX940-TGSPLIT-NEXT:    s_endpgm
7071;
7072; GFX11-WGP-LABEL: flat_workgroup_one_as_unordered_store:
7073; GFX11-WGP:       ; %bb.0: ; %entry
7074; GFX11-WGP-NEXT:    s_clause 0x1
7075; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7076; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
7077; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7078; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7079; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7080; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7081; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7082; GFX11-WGP-NEXT:    s_endpgm
7083;
7084; GFX11-CU-LABEL: flat_workgroup_one_as_unordered_store:
7085; GFX11-CU:       ; %bb.0: ; %entry
7086; GFX11-CU-NEXT:    s_clause 0x1
7087; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7088; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
7089; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7090; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7091; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7092; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7093; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7094; GFX11-CU-NEXT:    s_endpgm
7095    i32 %in, i32* %out) {
7096entry:
7097  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") unordered, align 4
7098  ret void
7099}
7100
7101define amdgpu_kernel void @flat_workgroup_one_as_monotonic_store(
7102; GFX7-LABEL: flat_workgroup_one_as_monotonic_store:
7103; GFX7:       ; %bb.0: ; %entry
7104; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7105; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7106; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7107; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7108; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7109; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7110; GFX7-NEXT:    flat_store_dword v[0:1], v2
7111; GFX7-NEXT:    s_endpgm
7112;
7113; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_store:
7114; GFX10-WGP:       ; %bb.0: ; %entry
7115; GFX10-WGP-NEXT:    s_clause 0x1
7116; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7117; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7118; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7119; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7120; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7121; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7122; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7123; GFX10-WGP-NEXT:    s_endpgm
7124;
7125; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_store:
7126; GFX10-CU:       ; %bb.0: ; %entry
7127; GFX10-CU-NEXT:    s_clause 0x1
7128; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7129; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7130; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7131; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7132; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7133; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7134; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7135; GFX10-CU-NEXT:    s_endpgm
7136;
7137; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_store:
7138; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7139; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
7140; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
7141; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7142; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7143; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7144; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7145; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7146; SKIP-CACHE-INV-NEXT:    s_endpgm
7147;
7148; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store:
7149; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7150; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7151; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7152; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7153; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7154; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7155; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7156; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7157;
7158; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store:
7159; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7160; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7161; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7162; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7163; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7164; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7165; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7166; GFX90A-TGSPLIT-NEXT:    s_endpgm
7167;
7168; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store:
7169; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7170; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7171; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7172; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7173; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7174; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7175; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
7176; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7177;
7178; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store:
7179; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7180; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7181; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7182; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7183; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7184; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7185; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
7186; GFX940-TGSPLIT-NEXT:    s_endpgm
7187;
7188; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_store:
7189; GFX11-WGP:       ; %bb.0: ; %entry
7190; GFX11-WGP-NEXT:    s_clause 0x1
7191; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7192; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
7193; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7194; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7195; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7196; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7197; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7198; GFX11-WGP-NEXT:    s_endpgm
7199;
7200; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_store:
7201; GFX11-CU:       ; %bb.0: ; %entry
7202; GFX11-CU-NEXT:    s_clause 0x1
7203; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7204; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
7205; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7206; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7207; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7208; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7209; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7210; GFX11-CU-NEXT:    s_endpgm
7211    i32 %in, i32* %out) {
7212entry:
7213  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") monotonic, align 4
7214  ret void
7215}
7216
7217define amdgpu_kernel void @flat_workgroup_one_as_release_store(
7218; GFX7-LABEL: flat_workgroup_one_as_release_store:
7219; GFX7:       ; %bb.0: ; %entry
7220; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7221; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7222; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7223; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7224; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7225; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7226; GFX7-NEXT:    flat_store_dword v[0:1], v2
7227; GFX7-NEXT:    s_endpgm
7228;
7229; GFX10-WGP-LABEL: flat_workgroup_one_as_release_store:
7230; GFX10-WGP:       ; %bb.0: ; %entry
7231; GFX10-WGP-NEXT:    s_clause 0x1
7232; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7233; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7234; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7235; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7236; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7237; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7238; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7239; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7240; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7241; GFX10-WGP-NEXT:    s_endpgm
7242;
7243; GFX10-CU-LABEL: flat_workgroup_one_as_release_store:
7244; GFX10-CU:       ; %bb.0: ; %entry
7245; GFX10-CU-NEXT:    s_clause 0x1
7246; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7247; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7248; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7249; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7250; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7251; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7252; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7253; GFX10-CU-NEXT:    s_endpgm
7254;
7255; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_store:
7256; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7257; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
7258; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
7259; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7260; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7261; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7262; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7263; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7264; SKIP-CACHE-INV-NEXT:    s_endpgm
7265;
7266; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_store:
7267; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7268; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7269; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7270; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7271; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7272; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7273; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7274; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7275;
7276; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_store:
7277; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7278; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7279; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7280; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7281; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7282; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7283; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7284; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7285; GFX90A-TGSPLIT-NEXT:    s_endpgm
7286;
7287; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_store:
7288; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7289; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7290; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7291; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7292; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7293; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7294; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
7295; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7296;
7297; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_store:
7298; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7299; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7300; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7301; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7302; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7303; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7304; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7305; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
7306; GFX940-TGSPLIT-NEXT:    s_endpgm
7307;
7308; GFX11-WGP-LABEL: flat_workgroup_one_as_release_store:
7309; GFX11-WGP:       ; %bb.0: ; %entry
7310; GFX11-WGP-NEXT:    s_clause 0x1
7311; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7312; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
7313; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7314; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7315; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7316; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
7317; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7318; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7319; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7320; GFX11-WGP-NEXT:    s_endpgm
7321;
7322; GFX11-CU-LABEL: flat_workgroup_one_as_release_store:
7323; GFX11-CU:       ; %bb.0: ; %entry
7324; GFX11-CU-NEXT:    s_clause 0x1
7325; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7326; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
7327; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7328; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7329; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7330; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7331; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7332; GFX11-CU-NEXT:    s_endpgm
7333    i32 %in, i32* %out) {
7334entry:
7335  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") release, align 4
7336  ret void
7337}
7338
7339define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_store(
7340; GFX7-LABEL: flat_workgroup_one_as_seq_cst_store:
7341; GFX7:       ; %bb.0: ; %entry
7342; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7343; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7344; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7345; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7346; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7347; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7348; GFX7-NEXT:    flat_store_dword v[0:1], v2
7349; GFX7-NEXT:    s_endpgm
7350;
7351; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_store:
7352; GFX10-WGP:       ; %bb.0: ; %entry
7353; GFX10-WGP-NEXT:    s_clause 0x1
7354; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7355; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7356; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7357; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7358; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7359; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7360; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7361; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7362; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7363; GFX10-WGP-NEXT:    s_endpgm
7364;
7365; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_store:
7366; GFX10-CU:       ; %bb.0: ; %entry
7367; GFX10-CU-NEXT:    s_clause 0x1
7368; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7369; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7370; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7371; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7372; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7373; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7374; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7375; GFX10-CU-NEXT:    s_endpgm
7376;
7377; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_store:
7378; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7379; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
7380; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
7381; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7382; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7383; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7384; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7385; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7386; SKIP-CACHE-INV-NEXT:    s_endpgm
7387;
7388; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store:
7389; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7390; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7391; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7392; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7393; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7394; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7395; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7396; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7397;
7398; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store:
7399; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7400; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7401; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7402; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7403; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7404; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7405; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7406; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7407; GFX90A-TGSPLIT-NEXT:    s_endpgm
7408;
7409; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store:
7410; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7411; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7412; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7413; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7414; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7415; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7416; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
7417; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7418;
7419; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store:
7420; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7421; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7422; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7423; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7424; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7425; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7426; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7427; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
7428; GFX940-TGSPLIT-NEXT:    s_endpgm
7429;
7430; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_store:
7431; GFX11-WGP:       ; %bb.0: ; %entry
7432; GFX11-WGP-NEXT:    s_clause 0x1
7433; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7434; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
7435; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7436; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7437; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7438; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
7439; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7440; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7441; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7442; GFX11-WGP-NEXT:    s_endpgm
7443;
7444; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_store:
7445; GFX11-CU:       ; %bb.0: ; %entry
7446; GFX11-CU-NEXT:    s_clause 0x1
7447; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7448; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
7449; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7450; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7451; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7452; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7453; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7454; GFX11-CU-NEXT:    s_endpgm
7455    i32 %in, i32* %out) {
7456entry:
7457  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") seq_cst, align 4
7458  ret void
7459}
7460
7461define amdgpu_kernel void @flat_workgroup_one_as_monotonic_atomicrmw(
7462; GFX7-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
7463; GFX7:       ; %bb.0: ; %entry
7464; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7465; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7466; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7467; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7468; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7469; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7470; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7471; GFX7-NEXT:    s_endpgm
7472;
7473; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
7474; GFX10-WGP:       ; %bb.0: ; %entry
7475; GFX10-WGP-NEXT:    s_clause 0x1
7476; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7477; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7478; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7479; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7480; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7481; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7482; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7483; GFX10-WGP-NEXT:    s_endpgm
7484;
7485; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
7486; GFX10-CU:       ; %bb.0: ; %entry
7487; GFX10-CU-NEXT:    s_clause 0x1
7488; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7489; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7490; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7491; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7492; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7493; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7494; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7495; GFX10-CU-NEXT:    s_endpgm
7496;
7497; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
7498; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7499; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7500; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7501; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7502; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7503; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7504; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7505; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7506; SKIP-CACHE-INV-NEXT:    s_endpgm
7507;
7508; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
7509; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7510; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7511; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7512; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7513; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7514; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7515; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7516; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7517;
7518; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
7519; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7520; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7521; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7522; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7523; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7524; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7525; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7526; GFX90A-TGSPLIT-NEXT:    s_endpgm
7527;
7528; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
7529; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7530; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7531; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7532; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7533; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7534; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7535; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7536; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7537;
7538; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
7539; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7540; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7541; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7542; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7543; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7544; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7545; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7546; GFX940-TGSPLIT-NEXT:    s_endpgm
7547;
7548; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
7549; GFX11-WGP:       ; %bb.0: ; %entry
7550; GFX11-WGP-NEXT:    s_clause 0x1
7551; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7552; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
7553; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7554; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7555; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7556; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7557; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7558; GFX11-WGP-NEXT:    s_endpgm
7559;
7560; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
7561; GFX11-CU:       ; %bb.0: ; %entry
7562; GFX11-CU-NEXT:    s_clause 0x1
7563; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7564; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7565; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7566; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7567; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7568; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7569; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7570; GFX11-CU-NEXT:    s_endpgm
7571    i32* %out, i32 %in) {
7572entry:
7573  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") monotonic
7574  ret void
7575}
7576
7577define amdgpu_kernel void @flat_workgroup_one_as_acquire_atomicrmw(
7578; GFX7-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
7579; GFX7:       ; %bb.0: ; %entry
7580; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7581; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7582; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7583; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7584; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7585; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7586; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7587; GFX7-NEXT:    s_endpgm
7588;
7589; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
7590; GFX10-WGP:       ; %bb.0: ; %entry
7591; GFX10-WGP-NEXT:    s_clause 0x1
7592; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7593; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7594; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7595; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7596; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7597; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7598; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7599; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7600; GFX10-WGP-NEXT:    buffer_gl0_inv
7601; GFX10-WGP-NEXT:    s_endpgm
7602;
7603; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
7604; GFX10-CU:       ; %bb.0: ; %entry
7605; GFX10-CU-NEXT:    s_clause 0x1
7606; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7607; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7608; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7609; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7610; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7611; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7612; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7613; GFX10-CU-NEXT:    s_endpgm
7614;
7615; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
7616; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7617; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7618; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7619; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7620; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7621; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7622; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7623; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7624; SKIP-CACHE-INV-NEXT:    s_endpgm
7625;
7626; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
7627; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7628; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7629; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7630; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7631; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7632; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7633; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7634; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7635;
7636; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
7637; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7638; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7639; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7640; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7641; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7642; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7643; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7644; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7645; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7646; GFX90A-TGSPLIT-NEXT:    s_endpgm
7647;
7648; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
7649; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7650; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7651; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7652; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7653; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7654; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7655; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7656; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7657;
7658; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
7659; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7660; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7661; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7662; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7663; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7664; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7665; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7666; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7667; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
7668; GFX940-TGSPLIT-NEXT:    s_endpgm
7669;
7670; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
7671; GFX11-WGP:       ; %bb.0: ; %entry
7672; GFX11-WGP-NEXT:    s_clause 0x1
7673; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7674; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
7675; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7676; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7677; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7678; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7679; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7680; GFX11-WGP-NEXT:    buffer_gl0_inv
7681; GFX11-WGP-NEXT:    s_endpgm
7682;
7683; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
7684; GFX11-CU:       ; %bb.0: ; %entry
7685; GFX11-CU-NEXT:    s_clause 0x1
7686; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7687; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7688; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7689; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7690; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7691; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7692; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7693; GFX11-CU-NEXT:    s_endpgm
7694    i32* %out, i32 %in) {
7695entry:
7696  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acquire
7697  ret void
7698}
7699
7700define amdgpu_kernel void @flat_workgroup_one_as_release_atomicrmw(
7701; GFX7-LABEL: flat_workgroup_one_as_release_atomicrmw:
7702; GFX7:       ; %bb.0: ; %entry
7703; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7704; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7705; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7706; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7707; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7708; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7709; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7710; GFX7-NEXT:    s_endpgm
7711;
7712; GFX10-WGP-LABEL: flat_workgroup_one_as_release_atomicrmw:
7713; GFX10-WGP:       ; %bb.0: ; %entry
7714; GFX10-WGP-NEXT:    s_clause 0x1
7715; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7716; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7717; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7718; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7719; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7720; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7721; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7722; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7723; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7724; GFX10-WGP-NEXT:    s_endpgm
7725;
7726; GFX10-CU-LABEL: flat_workgroup_one_as_release_atomicrmw:
7727; GFX10-CU:       ; %bb.0: ; %entry
7728; GFX10-CU-NEXT:    s_clause 0x1
7729; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7730; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7731; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7732; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7733; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7734; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7735; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7736; GFX10-CU-NEXT:    s_endpgm
7737;
7738; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_atomicrmw:
7739; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7740; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7741; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7742; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7743; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7744; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7745; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7746; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7747; SKIP-CACHE-INV-NEXT:    s_endpgm
7748;
7749; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw:
7750; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7751; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7752; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7753; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7754; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7755; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7756; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7757; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7758;
7759; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw:
7760; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7761; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7762; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7763; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7764; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7765; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7766; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7767; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7768; GFX90A-TGSPLIT-NEXT:    s_endpgm
7769;
7770; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw:
7771; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7772; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7773; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7774; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7775; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7776; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7777; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7778; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7779;
7780; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw:
7781; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7782; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7783; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7784; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7785; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7786; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7787; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7788; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7789; GFX940-TGSPLIT-NEXT:    s_endpgm
7790;
7791; GFX11-WGP-LABEL: flat_workgroup_one_as_release_atomicrmw:
7792; GFX11-WGP:       ; %bb.0: ; %entry
7793; GFX11-WGP-NEXT:    s_clause 0x1
7794; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7795; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
7796; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7797; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7798; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7799; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
7800; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7801; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7802; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7803; GFX11-WGP-NEXT:    s_endpgm
7804;
7805; GFX11-CU-LABEL: flat_workgroup_one_as_release_atomicrmw:
7806; GFX11-CU:       ; %bb.0: ; %entry
7807; GFX11-CU-NEXT:    s_clause 0x1
7808; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7809; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7810; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7811; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7812; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7813; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7814; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7815; GFX11-CU-NEXT:    s_endpgm
7816    i32* %out, i32 %in) {
7817entry:
7818  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") release
7819  ret void
7820}
7821
7822define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_atomicrmw(
7823; GFX7-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
7824; GFX7:       ; %bb.0: ; %entry
7825; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7826; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7827; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7828; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7829; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7830; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7831; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7832; GFX7-NEXT:    s_endpgm
7833;
7834; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
7835; GFX10-WGP:       ; %bb.0: ; %entry
7836; GFX10-WGP-NEXT:    s_clause 0x1
7837; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7838; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7839; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7840; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7841; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7842; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7843; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7844; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7845; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7846; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7847; GFX10-WGP-NEXT:    buffer_gl0_inv
7848; GFX10-WGP-NEXT:    s_endpgm
7849;
7850; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
7851; GFX10-CU:       ; %bb.0: ; %entry
7852; GFX10-CU-NEXT:    s_clause 0x1
7853; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7854; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7855; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7856; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7857; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7858; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7859; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7860; GFX10-CU-NEXT:    s_endpgm
7861;
7862; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
7863; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7864; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7865; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7866; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7867; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7868; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7869; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7870; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7871; SKIP-CACHE-INV-NEXT:    s_endpgm
7872;
7873; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
7874; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7875; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7876; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7877; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7878; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7879; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7880; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7881; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7882;
7883; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
7884; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7885; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7886; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7887; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7888; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7889; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7890; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7891; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7892; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7893; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7894; GFX90A-TGSPLIT-NEXT:    s_endpgm
7895;
7896; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
7897; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7898; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7899; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7900; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7901; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7902; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7903; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7904; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7905;
7906; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
7907; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7908; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7909; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7910; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7911; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7912; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7913; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7914; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7915; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7916; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
7917; GFX940-TGSPLIT-NEXT:    s_endpgm
7918;
7919; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
7920; GFX11-WGP:       ; %bb.0: ; %entry
7921; GFX11-WGP-NEXT:    s_clause 0x1
7922; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7923; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
7924; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7925; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7926; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7927; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
7928; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7929; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7930; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7931; GFX11-WGP-NEXT:    buffer_gl0_inv
7932; GFX11-WGP-NEXT:    s_endpgm
7933;
7934; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
7935; GFX11-CU:       ; %bb.0: ; %entry
7936; GFX11-CU-NEXT:    s_clause 0x1
7937; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7938; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7939; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7940; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7941; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7942; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7943; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7944; GFX11-CU-NEXT:    s_endpgm
7945    i32* %out, i32 %in) {
7946entry:
7947  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acq_rel
7948  ret void
7949}
7950
7951define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_atomicrmw(
7952; GFX7-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
7953; GFX7:       ; %bb.0: ; %entry
7954; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7955; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7956; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7957; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7958; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7959; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7960; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7961; GFX7-NEXT:    s_endpgm
7962;
7963; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
7964; GFX10-WGP:       ; %bb.0: ; %entry
7965; GFX10-WGP-NEXT:    s_clause 0x1
7966; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7967; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7968; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7969; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7970; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7971; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7972; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7973; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7974; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7975; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7976; GFX10-WGP-NEXT:    buffer_gl0_inv
7977; GFX10-WGP-NEXT:    s_endpgm
7978;
7979; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
7980; GFX10-CU:       ; %bb.0: ; %entry
7981; GFX10-CU-NEXT:    s_clause 0x1
7982; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7983; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7984; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7985; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7986; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7987; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7988; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7989; GFX10-CU-NEXT:    s_endpgm
7990;
7991; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
7992; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7993; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7994; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7995; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7996; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7997; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7998; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7999; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
8000; SKIP-CACHE-INV-NEXT:    s_endpgm
8001;
8002; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
8003; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8004; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8005; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8006; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8007; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8008; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8009; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8010; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8011;
8012; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
8013; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8014; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8015; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8016; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8017; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8018; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8019; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8020; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8021; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8022; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8023; GFX90A-TGSPLIT-NEXT:    s_endpgm
8024;
8025; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
8026; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8027; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8028; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8029; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8030; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8031; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8032; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8033; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8034;
8035; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
8036; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8037; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8038; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8039; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8040; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8041; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8042; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8043; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8044; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8045; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
8046; GFX940-TGSPLIT-NEXT:    s_endpgm
8047;
8048; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
8049; GFX11-WGP:       ; %bb.0: ; %entry
8050; GFX11-WGP-NEXT:    s_clause 0x1
8051; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8052; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8053; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8054; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8055; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8056; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8057; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8058; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8059; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8060; GFX11-WGP-NEXT:    buffer_gl0_inv
8061; GFX11-WGP-NEXT:    s_endpgm
8062;
8063; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
8064; GFX11-CU:       ; %bb.0: ; %entry
8065; GFX11-CU-NEXT:    s_clause 0x1
8066; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8067; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8068; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8069; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8070; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8071; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8072; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8073; GFX11-CU-NEXT:    s_endpgm
8074    i32* %out, i32 %in) {
8075entry:
8076  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") seq_cst
8077  ret void
8078}
8079
8080define amdgpu_kernel void @flat_workgroup_one_as_acquire_ret_atomicrmw(
8081; GFX7-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
8082; GFX7:       ; %bb.0: ; %entry
8083; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8084; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8085; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8086; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8087; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8088; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8089; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8090; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8091; GFX7-NEXT:    flat_store_dword v[0:1], v2
8092; GFX7-NEXT:    s_endpgm
8093;
8094; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
8095; GFX10-WGP:       ; %bb.0: ; %entry
8096; GFX10-WGP-NEXT:    s_clause 0x1
8097; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8098; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8099; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8100; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8101; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8102; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8103; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8104; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8105; GFX10-WGP-NEXT:    buffer_gl0_inv
8106; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8107; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8108; GFX10-WGP-NEXT:    s_endpgm
8109;
8110; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
8111; GFX10-CU:       ; %bb.0: ; %entry
8112; GFX10-CU-NEXT:    s_clause 0x1
8113; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8114; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8115; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8116; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8117; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8118; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8119; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8120; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8121; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8122; GFX10-CU-NEXT:    s_endpgm
8123;
8124; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
8125; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8126; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8127; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8128; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8129; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8130; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8131; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8132; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8133; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8134; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8135; SKIP-CACHE-INV-NEXT:    s_endpgm
8136;
8137; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
8138; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8139; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8140; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8141; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8142; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8143; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8144; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8145; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8146; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8147; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8148;
8149; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
8150; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8151; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8152; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8153; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8154; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8155; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8156; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8157; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8158; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8159; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8160; GFX90A-TGSPLIT-NEXT:    s_endpgm
8161;
8162; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
8163; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8164; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8165; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8166; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8167; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8168; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8169; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8170; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8171; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8172; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8173;
8174; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
8175; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8176; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8177; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8178; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8179; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8180; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8181; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8182; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8183; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
8184; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8185; GFX940-TGSPLIT-NEXT:    s_endpgm
8186;
8187; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
8188; GFX11-WGP:       ; %bb.0: ; %entry
8189; GFX11-WGP-NEXT:    s_clause 0x1
8190; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8191; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8192; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8193; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8194; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8195; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8196; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8197; GFX11-WGP-NEXT:    buffer_gl0_inv
8198; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8199; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8200; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8201; GFX11-WGP-NEXT:    s_endpgm
8202;
8203; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
8204; GFX11-CU:       ; %bb.0: ; %entry
8205; GFX11-CU-NEXT:    s_clause 0x1
8206; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8207; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8208; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8209; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8210; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8211; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8212; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8213; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8214; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8215; GFX11-CU-NEXT:    s_endpgm
8216    i32* %out, i32 %in) {
8217entry:
8218  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acquire
8219  store i32 %val, i32* %out, align 4
8220  ret void
8221}
8222
8223define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_ret_atomicrmw(
8224; GFX7-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
8225; GFX7:       ; %bb.0: ; %entry
8226; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8227; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8228; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8229; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8230; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8231; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8232; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8233; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8234; GFX7-NEXT:    flat_store_dword v[0:1], v2
8235; GFX7-NEXT:    s_endpgm
8236;
8237; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
8238; GFX10-WGP:       ; %bb.0: ; %entry
8239; GFX10-WGP-NEXT:    s_clause 0x1
8240; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8241; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8242; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8243; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8244; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8245; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8246; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8247; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8248; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8249; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8250; GFX10-WGP-NEXT:    buffer_gl0_inv
8251; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8252; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8253; GFX10-WGP-NEXT:    s_endpgm
8254;
8255; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
8256; GFX10-CU:       ; %bb.0: ; %entry
8257; GFX10-CU-NEXT:    s_clause 0x1
8258; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8259; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8260; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8261; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8262; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8263; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8264; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8265; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8266; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8267; GFX10-CU-NEXT:    s_endpgm
8268;
8269; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
8270; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8271; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8272; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8273; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8274; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8275; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8276; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8277; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8278; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8279; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8280; SKIP-CACHE-INV-NEXT:    s_endpgm
8281;
8282; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
8283; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8284; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8285; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8286; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8287; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8288; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8289; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8290; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8291; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8292; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8293;
8294; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
8295; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8296; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8297; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8298; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8299; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8300; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8301; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8302; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8303; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8304; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8305; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8306; GFX90A-TGSPLIT-NEXT:    s_endpgm
8307;
8308; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
8309; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8310; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8311; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8312; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8313; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8314; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8315; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8316; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8317; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8318; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8319;
8320; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
8321; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8322; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8323; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8324; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8325; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8326; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8327; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8328; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8329; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8330; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
8331; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8332; GFX940-TGSPLIT-NEXT:    s_endpgm
8333;
8334; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
8335; GFX11-WGP:       ; %bb.0: ; %entry
8336; GFX11-WGP-NEXT:    s_clause 0x1
8337; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8338; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8339; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8340; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8341; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8342; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8343; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8344; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8345; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8346; GFX11-WGP-NEXT:    buffer_gl0_inv
8347; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8348; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8349; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8350; GFX11-WGP-NEXT:    s_endpgm
8351;
8352; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
8353; GFX11-CU:       ; %bb.0: ; %entry
8354; GFX11-CU-NEXT:    s_clause 0x1
8355; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8356; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8357; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8358; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8359; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8360; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8361; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8362; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8363; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8364; GFX11-CU-NEXT:    s_endpgm
8365    i32* %out, i32 %in) {
8366entry:
8367  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acq_rel
8368  store i32 %val, i32* %out, align 4
8369  ret void
8370}
8371
8372define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_ret_atomicrmw(
8373; GFX7-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
8374; GFX7:       ; %bb.0: ; %entry
8375; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8376; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8377; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8378; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8379; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8380; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8381; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8382; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8383; GFX7-NEXT:    flat_store_dword v[0:1], v2
8384; GFX7-NEXT:    s_endpgm
8385;
8386; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
8387; GFX10-WGP:       ; %bb.0: ; %entry
8388; GFX10-WGP-NEXT:    s_clause 0x1
8389; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8390; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8391; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8392; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8393; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8394; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8395; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8396; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8397; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8398; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8399; GFX10-WGP-NEXT:    buffer_gl0_inv
8400; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8401; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8402; GFX10-WGP-NEXT:    s_endpgm
8403;
8404; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
8405; GFX10-CU:       ; %bb.0: ; %entry
8406; GFX10-CU-NEXT:    s_clause 0x1
8407; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8408; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8409; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8410; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8411; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8412; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8413; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8414; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8415; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8416; GFX10-CU-NEXT:    s_endpgm
8417;
8418; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
8419; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8420; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8421; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8422; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8423; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8424; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8425; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8426; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8427; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8428; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8429; SKIP-CACHE-INV-NEXT:    s_endpgm
8430;
8431; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
8432; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8433; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8434; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8435; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8436; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8437; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8438; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8439; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8440; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8441; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8442;
8443; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
8444; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8445; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8446; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8447; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8448; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8449; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8450; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8451; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8452; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8453; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8454; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8455; GFX90A-TGSPLIT-NEXT:    s_endpgm
8456;
8457; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
8458; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8459; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8460; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8461; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8462; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8463; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8464; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8465; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8466; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8467; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8468;
8469; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
8470; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8471; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8472; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8473; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8474; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8475; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8476; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8477; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8478; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8479; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
8480; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8481; GFX940-TGSPLIT-NEXT:    s_endpgm
8482;
8483; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
8484; GFX11-WGP:       ; %bb.0: ; %entry
8485; GFX11-WGP-NEXT:    s_clause 0x1
8486; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8487; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8488; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8489; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8490; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8491; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8492; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8493; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8494; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8495; GFX11-WGP-NEXT:    buffer_gl0_inv
8496; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8497; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8498; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8499; GFX11-WGP-NEXT:    s_endpgm
8500;
8501; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
8502; GFX11-CU:       ; %bb.0: ; %entry
8503; GFX11-CU-NEXT:    s_clause 0x1
8504; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8505; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8506; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8507; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8508; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8509; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8510; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8511; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8512; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8513; GFX11-CU-NEXT:    s_endpgm
8514    i32* %out, i32 %in) {
8515entry:
8516  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") seq_cst
8517  store i32 %val, i32* %out, align 4
8518  ret void
8519}
8520
8521define amdgpu_kernel void @flat_workgroup_one_as_monotonic_monotonic_cmpxchg(
8522; GFX7-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
8523; GFX7:       ; %bb.0: ; %entry
8524; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8525; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8526; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8527; GFX7-NEXT:    s_add_u32 s0, s0, 16
8528; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8529; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8530; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8531; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8532; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8533; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8534; GFX7-NEXT:    s_endpgm
8535;
8536; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
8537; GFX10-WGP:       ; %bb.0: ; %entry
8538; GFX10-WGP-NEXT:    s_clause 0x1
8539; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8540; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8541; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8542; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8543; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8544; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8545; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8546; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8547; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8548; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8549; GFX10-WGP-NEXT:    s_endpgm
8550;
8551; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
8552; GFX10-CU:       ; %bb.0: ; %entry
8553; GFX10-CU-NEXT:    s_clause 0x1
8554; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8555; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8556; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8557; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8558; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8559; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8560; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8561; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8562; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8563; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8564; GFX10-CU-NEXT:    s_endpgm
8565;
8566; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
8567; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8568; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8569; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8570; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8571; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8572; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8573; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8574; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8575; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8576; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8577; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8578; SKIP-CACHE-INV-NEXT:    s_endpgm
8579;
8580; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
8581; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8582; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8583; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8584; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8585; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8586; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8587; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8588; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8589;
8590; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
8591; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8592; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8593; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8594; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8595; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8596; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8597; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8598; GFX90A-TGSPLIT-NEXT:    s_endpgm
8599;
8600; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
8601; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8602; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8603; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8604; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8605; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8606; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8607; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8608; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8609;
8610; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
8611; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8612; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8613; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8614; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8615; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8616; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8617; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8618; GFX940-TGSPLIT-NEXT:    s_endpgm
8619;
8620; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
8621; GFX11-WGP:       ; %bb.0: ; %entry
8622; GFX11-WGP-NEXT:    s_clause 0x1
8623; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8624; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8625; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8626; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8627; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
8628; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8629; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8630; GFX11-WGP-NEXT:    s_endpgm
8631;
8632; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
8633; GFX11-CU:       ; %bb.0: ; %entry
8634; GFX11-CU-NEXT:    s_clause 0x1
8635; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8636; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8637; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8638; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8639; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
8640; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8641; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8642; GFX11-CU-NEXT:    s_endpgm
8643    i32* %out, i32 %in, i32 %old) {
8644entry:
8645  %gep = getelementptr i32, i32* %out, i32 4
8646  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic
8647  ret void
8648}
8649
8650define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_cmpxchg(
8651; GFX7-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
8652; GFX7:       ; %bb.0: ; %entry
8653; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8654; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8655; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8656; GFX7-NEXT:    s_add_u32 s0, s0, 16
8657; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8658; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8659; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8660; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8661; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8662; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8663; GFX7-NEXT:    s_endpgm
8664;
8665; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
8666; GFX10-WGP:       ; %bb.0: ; %entry
8667; GFX10-WGP-NEXT:    s_clause 0x1
8668; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8669; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8670; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8671; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8672; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8673; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8674; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8675; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8676; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8677; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8678; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8679; GFX10-WGP-NEXT:    buffer_gl0_inv
8680; GFX10-WGP-NEXT:    s_endpgm
8681;
8682; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
8683; GFX10-CU:       ; %bb.0: ; %entry
8684; GFX10-CU-NEXT:    s_clause 0x1
8685; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8686; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8687; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8688; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8689; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8690; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8691; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8692; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8693; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8694; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8695; GFX10-CU-NEXT:    s_endpgm
8696;
8697; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
8698; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8699; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8700; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8701; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8702; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8703; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8704; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8705; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8706; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8707; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8708; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8709; SKIP-CACHE-INV-NEXT:    s_endpgm
8710;
8711; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
8712; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8713; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8714; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8715; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8716; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8717; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8718; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8719; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8720;
8721; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
8722; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8723; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8724; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8725; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8726; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8727; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8728; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8729; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8730; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8731; GFX90A-TGSPLIT-NEXT:    s_endpgm
8732;
8733; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
8734; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8735; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8736; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8737; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8738; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8739; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8740; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8741; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8742;
8743; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
8744; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8745; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8746; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8747; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8748; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8749; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8750; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8751; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8752; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
8753; GFX940-TGSPLIT-NEXT:    s_endpgm
8754;
8755; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
8756; GFX11-WGP:       ; %bb.0: ; %entry
8757; GFX11-WGP-NEXT:    s_clause 0x1
8758; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8759; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8760; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8761; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8762; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
8763; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8764; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8765; GFX11-WGP-NEXT:    buffer_gl0_inv
8766; GFX11-WGP-NEXT:    s_endpgm
8767;
8768; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
8769; GFX11-CU:       ; %bb.0: ; %entry
8770; GFX11-CU-NEXT:    s_clause 0x1
8771; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8772; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8773; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8774; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8775; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
8776; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8777; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8778; GFX11-CU-NEXT:    s_endpgm
8779    i32* %out, i32 %in, i32 %old) {
8780entry:
8781  %gep = getelementptr i32, i32* %out, i32 4
8782  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
8783  ret void
8784}
8785
8786define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_cmpxchg(
8787; GFX7-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
8788; GFX7:       ; %bb.0: ; %entry
8789; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8790; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8791; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8792; GFX7-NEXT:    s_add_u32 s0, s0, 16
8793; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8794; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8795; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8796; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8797; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8798; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8799; GFX7-NEXT:    s_endpgm
8800;
8801; GFX10-WGP-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
8802; GFX10-WGP:       ; %bb.0: ; %entry
8803; GFX10-WGP-NEXT:    s_clause 0x1
8804; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8805; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8806; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8807; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8808; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8809; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8810; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8811; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8812; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8813; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8814; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8815; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8816; GFX10-WGP-NEXT:    s_endpgm
8817;
8818; GFX10-CU-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
8819; GFX10-CU:       ; %bb.0: ; %entry
8820; GFX10-CU-NEXT:    s_clause 0x1
8821; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8822; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8823; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8824; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8825; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8826; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8827; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8828; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8829; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8830; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8831; GFX10-CU-NEXT:    s_endpgm
8832;
8833; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
8834; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8835; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8836; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8837; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8838; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8839; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8840; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8841; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8842; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8843; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8844; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8845; SKIP-CACHE-INV-NEXT:    s_endpgm
8846;
8847; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
8848; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8849; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8850; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8851; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8852; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8853; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8854; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8855; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8856;
8857; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
8858; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8859; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8860; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8861; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8862; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8863; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8864; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8865; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8866; GFX90A-TGSPLIT-NEXT:    s_endpgm
8867;
8868; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
8869; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8870; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8871; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8872; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8873; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8874; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8875; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8876; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8877;
8878; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
8879; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8880; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8881; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8882; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8883; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8884; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8885; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8886; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8887; GFX940-TGSPLIT-NEXT:    s_endpgm
8888;
8889; GFX11-WGP-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
8890; GFX11-WGP:       ; %bb.0: ; %entry
8891; GFX11-WGP-NEXT:    s_clause 0x1
8892; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8893; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8894; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8895; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8896; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
8897; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8898; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8899; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8900; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8901; GFX11-WGP-NEXT:    s_endpgm
8902;
8903; GFX11-CU-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
8904; GFX11-CU:       ; %bb.0: ; %entry
8905; GFX11-CU-NEXT:    s_clause 0x1
8906; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8907; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8908; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8909; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8910; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
8911; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8912; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8913; GFX11-CU-NEXT:    s_endpgm
8914    i32* %out, i32 %in, i32 %old) {
8915entry:
8916  %gep = getelementptr i32, i32* %out, i32 4
8917  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic
8918  ret void
8919}
8920
8921define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_cmpxchg(
8922; GFX7-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
8923; GFX7:       ; %bb.0: ; %entry
8924; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8925; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8926; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8927; GFX7-NEXT:    s_add_u32 s0, s0, 16
8928; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8929; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8930; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8931; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8932; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8933; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8934; GFX7-NEXT:    s_endpgm
8935;
8936; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
8937; GFX10-WGP:       ; %bb.0: ; %entry
8938; GFX10-WGP-NEXT:    s_clause 0x1
8939; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8940; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8941; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8942; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8943; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8944; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8945; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8946; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8947; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8948; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8949; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8950; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8951; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8952; GFX10-WGP-NEXT:    buffer_gl0_inv
8953; GFX10-WGP-NEXT:    s_endpgm
8954;
8955; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
8956; GFX10-CU:       ; %bb.0: ; %entry
8957; GFX10-CU-NEXT:    s_clause 0x1
8958; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8959; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8960; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8961; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8962; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8963; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8964; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8965; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8966; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8967; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8968; GFX10-CU-NEXT:    s_endpgm
8969;
8970; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
8971; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8972; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8973; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8974; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8975; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8976; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8977; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8978; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8979; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8980; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8981; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8982; SKIP-CACHE-INV-NEXT:    s_endpgm
8983;
8984; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
8985; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8986; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8987; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8988; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8989; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8990; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8991; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8992; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8993;
8994; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
8995; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8996; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8997; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8998; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8999; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9000; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9001; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9002; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9003; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9004; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9005; GFX90A-TGSPLIT-NEXT:    s_endpgm
9006;
9007; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
9008; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9009; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9010; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9011; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9012; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9013; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9014; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9015; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9016;
9017; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
9018; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9019; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9020; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9021; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9022; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9023; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9024; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9025; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9026; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9027; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
9028; GFX940-TGSPLIT-NEXT:    s_endpgm
9029;
9030; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
9031; GFX11-WGP:       ; %bb.0: ; %entry
9032; GFX11-WGP-NEXT:    s_clause 0x1
9033; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9034; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9035; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9036; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9037; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9038; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9039; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9040; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9041; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9042; GFX11-WGP-NEXT:    buffer_gl0_inv
9043; GFX11-WGP-NEXT:    s_endpgm
9044;
9045; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
9046; GFX11-CU:       ; %bb.0: ; %entry
9047; GFX11-CU-NEXT:    s_clause 0x1
9048; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9049; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9050; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9051; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9052; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9053; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9054; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9055; GFX11-CU-NEXT:    s_endpgm
9056    i32* %out, i32 %in, i32 %old) {
9057entry:
9058  %gep = getelementptr i32, i32* %out, i32 4
9059  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
9060  ret void
9061}
9062
9063define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_cmpxchg(
9064; GFX7-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
9065; GFX7:       ; %bb.0: ; %entry
9066; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9067; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9068; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9069; GFX7-NEXT:    s_add_u32 s0, s0, 16
9070; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9071; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9072; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9073; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9074; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9075; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9076; GFX7-NEXT:    s_endpgm
9077;
9078; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
9079; GFX10-WGP:       ; %bb.0: ; %entry
9080; GFX10-WGP-NEXT:    s_clause 0x1
9081; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9082; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9083; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9084; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9085; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9086; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9087; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9088; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9089; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9090; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9091; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9092; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9093; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9094; GFX10-WGP-NEXT:    buffer_gl0_inv
9095; GFX10-WGP-NEXT:    s_endpgm
9096;
9097; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
9098; GFX10-CU:       ; %bb.0: ; %entry
9099; GFX10-CU-NEXT:    s_clause 0x1
9100; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9101; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9102; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9103; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9104; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9105; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9106; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9107; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9108; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9109; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9110; GFX10-CU-NEXT:    s_endpgm
9111;
9112; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
9113; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9114; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9115; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9116; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9117; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9118; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9119; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9120; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9121; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9122; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9123; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9124; SKIP-CACHE-INV-NEXT:    s_endpgm
9125;
9126; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
9127; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9128; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9129; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9130; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9131; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9132; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9133; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9134; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9135;
9136; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
9137; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9138; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9139; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9140; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9141; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9142; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9143; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9144; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9145; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9146; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9147; GFX90A-TGSPLIT-NEXT:    s_endpgm
9148;
9149; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
9150; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9151; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9152; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9153; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9154; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9155; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9156; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9157; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9158;
9159; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
9160; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9161; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9162; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9163; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9164; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9165; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9166; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9167; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9168; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9169; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
9170; GFX940-TGSPLIT-NEXT:    s_endpgm
9171;
9172; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
9173; GFX11-WGP:       ; %bb.0: ; %entry
9174; GFX11-WGP-NEXT:    s_clause 0x1
9175; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9176; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9177; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9178; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9179; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9180; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9181; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9182; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9183; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9184; GFX11-WGP-NEXT:    buffer_gl0_inv
9185; GFX11-WGP-NEXT:    s_endpgm
9186;
9187; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
9188; GFX11-CU:       ; %bb.0: ; %entry
9189; GFX11-CU-NEXT:    s_clause 0x1
9190; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9191; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9192; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9193; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9194; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9195; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9196; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9197; GFX11-CU-NEXT:    s_endpgm
9198    i32* %out, i32 %in, i32 %old) {
9199entry:
9200  %gep = getelementptr i32, i32* %out, i32 4
9201  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
9202  ret void
9203}
9204
9205define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_cmpxchg(
9206; GFX7-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
9207; GFX7:       ; %bb.0: ; %entry
9208; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9209; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9210; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9211; GFX7-NEXT:    s_add_u32 s0, s0, 16
9212; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9213; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9214; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9215; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9216; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9217; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9218; GFX7-NEXT:    s_endpgm
9219;
9220; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
9221; GFX10-WGP:       ; %bb.0: ; %entry
9222; GFX10-WGP-NEXT:    s_clause 0x1
9223; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9224; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9225; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9226; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9227; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9228; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9229; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9230; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9231; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9232; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9233; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9234; GFX10-WGP-NEXT:    buffer_gl0_inv
9235; GFX10-WGP-NEXT:    s_endpgm
9236;
9237; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
9238; GFX10-CU:       ; %bb.0: ; %entry
9239; GFX10-CU-NEXT:    s_clause 0x1
9240; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9241; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9242; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9243; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9244; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9245; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9246; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9247; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9248; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9249; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9250; GFX10-CU-NEXT:    s_endpgm
9251;
9252; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
9253; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9254; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9255; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9256; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9257; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9258; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9259; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9260; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9261; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9262; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9263; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9264; SKIP-CACHE-INV-NEXT:    s_endpgm
9265;
9266; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
9267; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9268; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9269; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9270; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9271; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9272; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9273; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9274; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9275;
9276; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
9277; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9278; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9279; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9280; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9281; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9282; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9283; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9284; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9285; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9286; GFX90A-TGSPLIT-NEXT:    s_endpgm
9287;
9288; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
9289; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9290; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9291; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9292; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9293; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9294; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9295; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9296; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9297;
9298; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
9299; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9300; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9301; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9302; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9303; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9304; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9305; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9306; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9307; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
9308; GFX940-TGSPLIT-NEXT:    s_endpgm
9309;
9310; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
9311; GFX11-WGP:       ; %bb.0: ; %entry
9312; GFX11-WGP-NEXT:    s_clause 0x1
9313; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9314; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9315; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9316; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9317; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9318; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9319; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9320; GFX11-WGP-NEXT:    buffer_gl0_inv
9321; GFX11-WGP-NEXT:    s_endpgm
9322;
9323; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
9324; GFX11-CU:       ; %bb.0: ; %entry
9325; GFX11-CU-NEXT:    s_clause 0x1
9326; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9327; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9328; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9329; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9330; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9331; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9332; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9333; GFX11-CU-NEXT:    s_endpgm
9334    i32* %out, i32 %in, i32 %old) {
9335entry:
9336  %gep = getelementptr i32, i32* %out, i32 4
9337  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic acquire
9338  ret void
9339}
9340
9341define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_cmpxchg(
9342; GFX7-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
9343; GFX7:       ; %bb.0: ; %entry
9344; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9345; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9346; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9347; GFX7-NEXT:    s_add_u32 s0, s0, 16
9348; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9349; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9350; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9351; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9352; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9353; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9354; GFX7-NEXT:    s_endpgm
9355;
9356; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
9357; GFX10-WGP:       ; %bb.0: ; %entry
9358; GFX10-WGP-NEXT:    s_clause 0x1
9359; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9360; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9361; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9362; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9363; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9364; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9365; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9366; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9367; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9368; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9369; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9370; GFX10-WGP-NEXT:    buffer_gl0_inv
9371; GFX10-WGP-NEXT:    s_endpgm
9372;
9373; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
9374; GFX10-CU:       ; %bb.0: ; %entry
9375; GFX10-CU-NEXT:    s_clause 0x1
9376; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9377; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9378; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9379; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9380; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9381; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9382; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9383; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9384; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9385; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9386; GFX10-CU-NEXT:    s_endpgm
9387;
9388; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
9389; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9390; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9391; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9392; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9393; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9394; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9395; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9396; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9397; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9398; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9399; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9400; SKIP-CACHE-INV-NEXT:    s_endpgm
9401;
9402; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
9403; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9404; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9405; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9406; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9407; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9408; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9409; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9410; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9411;
9412; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
9413; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9414; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9415; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9416; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9417; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9418; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9419; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9420; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9421; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9422; GFX90A-TGSPLIT-NEXT:    s_endpgm
9423;
9424; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
9425; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9426; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9427; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9428; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9429; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9430; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9431; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9432; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9433;
9434; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
9435; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9436; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9437; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9438; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9439; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9440; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9441; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9442; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9443; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
9444; GFX940-TGSPLIT-NEXT:    s_endpgm
9445;
9446; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
9447; GFX11-WGP:       ; %bb.0: ; %entry
9448; GFX11-WGP-NEXT:    s_clause 0x1
9449; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9450; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9451; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9452; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9453; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9454; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9455; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9456; GFX11-WGP-NEXT:    buffer_gl0_inv
9457; GFX11-WGP-NEXT:    s_endpgm
9458;
9459; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
9460; GFX11-CU:       ; %bb.0: ; %entry
9461; GFX11-CU-NEXT:    s_clause 0x1
9462; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9463; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9464; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9465; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9466; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9467; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9468; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9469; GFX11-CU-NEXT:    s_endpgm
9470    i32* %out, i32 %in, i32 %old) {
9471entry:
9472  %gep = getelementptr i32, i32* %out, i32 4
9473  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
9474  ret void
9475}
9476
9477define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_cmpxchg(
9478; GFX7-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
9479; GFX7:       ; %bb.0: ; %entry
9480; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9481; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9482; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9483; GFX7-NEXT:    s_add_u32 s0, s0, 16
9484; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9485; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9486; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9487; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9488; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9489; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9490; GFX7-NEXT:    s_endpgm
9491;
9492; GFX10-WGP-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
9493; GFX10-WGP:       ; %bb.0: ; %entry
9494; GFX10-WGP-NEXT:    s_clause 0x1
9495; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9496; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9497; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9498; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9499; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9500; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9501; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9502; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9503; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9504; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9505; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9506; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9507; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9508; GFX10-WGP-NEXT:    buffer_gl0_inv
9509; GFX10-WGP-NEXT:    s_endpgm
9510;
9511; GFX10-CU-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
9512; GFX10-CU:       ; %bb.0: ; %entry
9513; GFX10-CU-NEXT:    s_clause 0x1
9514; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9515; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9516; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9517; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9518; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9519; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9520; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9521; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9522; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9523; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9524; GFX10-CU-NEXT:    s_endpgm
9525;
9526; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
9527; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9528; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9529; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9530; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9531; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9532; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9533; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9534; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9535; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9536; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9537; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9538; SKIP-CACHE-INV-NEXT:    s_endpgm
9539;
9540; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
9541; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9542; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9543; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9544; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9545; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9546; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9547; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9548; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9549;
9550; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
9551; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9552; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9553; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9554; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9555; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9556; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9557; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9558; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9559; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9560; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9561; GFX90A-TGSPLIT-NEXT:    s_endpgm
9562;
9563; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
9564; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9565; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9566; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9567; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9568; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9569; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9570; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9571; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9572;
9573; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
9574; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9575; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9576; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9577; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9578; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9579; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9580; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9581; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9582; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9583; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
9584; GFX940-TGSPLIT-NEXT:    s_endpgm
9585;
9586; GFX11-WGP-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
9587; GFX11-WGP:       ; %bb.0: ; %entry
9588; GFX11-WGP-NEXT:    s_clause 0x1
9589; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9590; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9591; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9592; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9593; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9594; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9595; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9596; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9597; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9598; GFX11-WGP-NEXT:    buffer_gl0_inv
9599; GFX11-WGP-NEXT:    s_endpgm
9600;
9601; GFX11-CU-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
9602; GFX11-CU:       ; %bb.0: ; %entry
9603; GFX11-CU-NEXT:    s_clause 0x1
9604; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9605; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9606; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9607; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9608; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9609; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9610; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9611; GFX11-CU-NEXT:    s_endpgm
9612    i32* %out, i32 %in, i32 %old) {
9613entry:
9614  %gep = getelementptr i32, i32* %out, i32 4
9615  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
9616  ret void
9617}
9618
9619define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_cmpxchg(
9620; GFX7-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
9621; GFX7:       ; %bb.0: ; %entry
9622; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9623; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9624; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9625; GFX7-NEXT:    s_add_u32 s0, s0, 16
9626; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9627; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9628; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9629; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9630; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9631; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9632; GFX7-NEXT:    s_endpgm
9633;
9634; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
9635; GFX10-WGP:       ; %bb.0: ; %entry
9636; GFX10-WGP-NEXT:    s_clause 0x1
9637; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9638; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9639; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9640; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9641; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9642; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9643; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9644; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9645; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9646; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9647; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9648; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9649; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9650; GFX10-WGP-NEXT:    buffer_gl0_inv
9651; GFX10-WGP-NEXT:    s_endpgm
9652;
9653; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
9654; GFX10-CU:       ; %bb.0: ; %entry
9655; GFX10-CU-NEXT:    s_clause 0x1
9656; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9657; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9658; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9659; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9660; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9661; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9662; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9663; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9664; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9665; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9666; GFX10-CU-NEXT:    s_endpgm
9667;
9668; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
9669; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9670; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9671; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9672; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9673; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9674; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9675; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9676; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9677; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9678; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9679; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9680; SKIP-CACHE-INV-NEXT:    s_endpgm
9681;
9682; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
9683; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9684; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9685; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9686; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9687; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9688; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9689; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9690; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9691;
9692; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
9693; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9694; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9695; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9696; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9697; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9698; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9699; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9700; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9701; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9702; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9703; GFX90A-TGSPLIT-NEXT:    s_endpgm
9704;
9705; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
9706; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9707; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9708; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9709; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9710; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9711; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9712; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9713; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9714;
9715; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
9716; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9717; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9718; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9719; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9720; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9721; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9722; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9723; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9724; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9725; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
9726; GFX940-TGSPLIT-NEXT:    s_endpgm
9727;
9728; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
9729; GFX11-WGP:       ; %bb.0: ; %entry
9730; GFX11-WGP-NEXT:    s_clause 0x1
9731; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9732; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9733; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9734; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9735; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9736; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9737; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9738; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9739; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9740; GFX11-WGP-NEXT:    buffer_gl0_inv
9741; GFX11-WGP-NEXT:    s_endpgm
9742;
9743; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
9744; GFX11-CU:       ; %bb.0: ; %entry
9745; GFX11-CU-NEXT:    s_clause 0x1
9746; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9747; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9748; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9749; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9750; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9751; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9752; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9753; GFX11-CU-NEXT:    s_endpgm
9754    i32* %out, i32 %in, i32 %old) {
9755entry:
9756  %gep = getelementptr i32, i32* %out, i32 4
9757  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
9758  ret void
9759}
9760
9761define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_cmpxchg(
9762; GFX7-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
9763; GFX7:       ; %bb.0: ; %entry
9764; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9765; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9766; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9767; GFX7-NEXT:    s_add_u32 s0, s0, 16
9768; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9769; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9770; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9771; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9772; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9773; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9774; GFX7-NEXT:    s_endpgm
9775;
9776; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
9777; GFX10-WGP:       ; %bb.0: ; %entry
9778; GFX10-WGP-NEXT:    s_clause 0x1
9779; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9780; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9781; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9782; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9783; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9784; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9785; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9786; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9787; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9788; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9789; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9790; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9791; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9792; GFX10-WGP-NEXT:    buffer_gl0_inv
9793; GFX10-WGP-NEXT:    s_endpgm
9794;
9795; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
9796; GFX10-CU:       ; %bb.0: ; %entry
9797; GFX10-CU-NEXT:    s_clause 0x1
9798; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9799; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9800; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9801; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9802; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9803; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9804; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9805; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9806; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9807; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9808; GFX10-CU-NEXT:    s_endpgm
9809;
9810; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
9811; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9812; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9813; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9814; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9815; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9816; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9817; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9818; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9819; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9820; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9821; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9822; SKIP-CACHE-INV-NEXT:    s_endpgm
9823;
9824; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
9825; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9826; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9827; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9828; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9829; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9830; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9831; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9832; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9833;
9834; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
9835; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9836; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9837; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9838; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9839; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9840; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9841; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9842; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9843; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9844; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9845; GFX90A-TGSPLIT-NEXT:    s_endpgm
9846;
9847; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
9848; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9849; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9850; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9851; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9852; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9853; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9854; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9855; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9856;
9857; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
9858; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9859; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9860; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9861; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9862; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9863; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9864; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9865; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9866; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9867; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
9868; GFX940-TGSPLIT-NEXT:    s_endpgm
9869;
9870; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
9871; GFX11-WGP:       ; %bb.0: ; %entry
9872; GFX11-WGP-NEXT:    s_clause 0x1
9873; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9874; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9875; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9876; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9877; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9878; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9879; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9880; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9881; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9882; GFX11-WGP-NEXT:    buffer_gl0_inv
9883; GFX11-WGP-NEXT:    s_endpgm
9884;
9885; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
9886; GFX11-CU:       ; %bb.0: ; %entry
9887; GFX11-CU-NEXT:    s_clause 0x1
9888; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9889; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9890; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9891; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9892; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9893; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9894; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9895; GFX11-CU-NEXT:    s_endpgm
9896    i32* %out, i32 %in, i32 %old) {
9897entry:
9898  %gep = getelementptr i32, i32* %out, i32 4
9899  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
9900  ret void
9901}
9902
9903define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_cmpxchg(
9904; GFX7-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
9905; GFX7:       ; %bb.0: ; %entry
9906; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9907; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9908; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9909; GFX7-NEXT:    s_add_u32 s0, s0, 16
9910; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9911; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9912; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9913; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9914; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9915; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9916; GFX7-NEXT:    s_endpgm
9917;
9918; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
9919; GFX10-WGP:       ; %bb.0: ; %entry
9920; GFX10-WGP-NEXT:    s_clause 0x1
9921; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9922; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9923; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9924; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9925; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9926; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9927; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9928; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9929; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9930; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9931; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9932; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9933; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9934; GFX10-WGP-NEXT:    buffer_gl0_inv
9935; GFX10-WGP-NEXT:    s_endpgm
9936;
9937; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
9938; GFX10-CU:       ; %bb.0: ; %entry
9939; GFX10-CU-NEXT:    s_clause 0x1
9940; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9941; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9942; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9943; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9944; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9945; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9946; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9947; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9948; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9949; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9950; GFX10-CU-NEXT:    s_endpgm
9951;
9952; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
9953; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9954; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9955; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9956; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9957; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9958; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9959; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9960; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9961; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9962; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9963; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9964; SKIP-CACHE-INV-NEXT:    s_endpgm
9965;
9966; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
9967; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9968; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9969; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9970; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9971; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9972; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9973; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9974; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9975;
9976; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
9977; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9978; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9979; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9980; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9981; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9982; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9983; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9984; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9985; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9986; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9987; GFX90A-TGSPLIT-NEXT:    s_endpgm
9988;
9989; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
9990; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9991; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9992; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9993; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9994; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9995; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9996; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9997; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9998;
9999; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
10000; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10001; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10002; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10003; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10004; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10005; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10006; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10007; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10008; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10009; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
10010; GFX940-TGSPLIT-NEXT:    s_endpgm
10011;
10012; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
10013; GFX11-WGP:       ; %bb.0: ; %entry
10014; GFX11-WGP-NEXT:    s_clause 0x1
10015; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10016; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10017; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10018; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10019; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10020; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10021; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10022; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10023; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10024; GFX11-WGP-NEXT:    buffer_gl0_inv
10025; GFX11-WGP-NEXT:    s_endpgm
10026;
10027; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
10028; GFX11-CU:       ; %bb.0: ; %entry
10029; GFX11-CU-NEXT:    s_clause 0x1
10030; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10031; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10032; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10033; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10034; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10035; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10036; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10037; GFX11-CU-NEXT:    s_endpgm
10038    i32* %out, i32 %in, i32 %old) {
10039entry:
10040  %gep = getelementptr i32, i32* %out, i32 4
10041  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic seq_cst
10042  ret void
10043}
10044
10045define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_cmpxchg(
10046; GFX7-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
10047; GFX7:       ; %bb.0: ; %entry
10048; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10049; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10050; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10051; GFX7-NEXT:    s_add_u32 s0, s0, 16
10052; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10053; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10054; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10055; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10056; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10057; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10058; GFX7-NEXT:    s_endpgm
10059;
10060; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
10061; GFX10-WGP:       ; %bb.0: ; %entry
10062; GFX10-WGP-NEXT:    s_clause 0x1
10063; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10064; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10065; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10066; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10067; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10068; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10069; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10070; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10071; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10072; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10073; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10074; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10075; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10076; GFX10-WGP-NEXT:    buffer_gl0_inv
10077; GFX10-WGP-NEXT:    s_endpgm
10078;
10079; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
10080; GFX10-CU:       ; %bb.0: ; %entry
10081; GFX10-CU-NEXT:    s_clause 0x1
10082; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10083; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10084; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10085; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10086; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10087; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10088; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10089; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10090; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10091; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10092; GFX10-CU-NEXT:    s_endpgm
10093;
10094; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
10095; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10096; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10097; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10098; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10099; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10100; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10101; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10102; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10103; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10104; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10105; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10106; SKIP-CACHE-INV-NEXT:    s_endpgm
10107;
10108; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
10109; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10110; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10111; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10112; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10113; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10114; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10115; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10116; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10117;
10118; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
10119; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10120; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10121; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10122; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10123; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10124; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10125; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10126; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10127; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10128; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10129; GFX90A-TGSPLIT-NEXT:    s_endpgm
10130;
10131; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
10132; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10133; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10134; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10135; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10136; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10137; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10138; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10139; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10140;
10141; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
10142; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10143; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10144; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10145; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10146; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10147; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10148; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10149; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10150; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10151; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
10152; GFX940-TGSPLIT-NEXT:    s_endpgm
10153;
10154; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
10155; GFX11-WGP:       ; %bb.0: ; %entry
10156; GFX11-WGP-NEXT:    s_clause 0x1
10157; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10158; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10159; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10160; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10161; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10162; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10163; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10164; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10165; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10166; GFX11-WGP-NEXT:    buffer_gl0_inv
10167; GFX11-WGP-NEXT:    s_endpgm
10168;
10169; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
10170; GFX11-CU:       ; %bb.0: ; %entry
10171; GFX11-CU-NEXT:    s_clause 0x1
10172; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10173; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10174; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10175; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10176; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10177; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10178; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10179; GFX11-CU-NEXT:    s_endpgm
10180    i32* %out, i32 %in, i32 %old) {
10181entry:
10182  %gep = getelementptr i32, i32* %out, i32 4
10183  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire seq_cst
10184  ret void
10185}
10186
10187define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_cmpxchg(
10188; GFX7-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
10189; GFX7:       ; %bb.0: ; %entry
10190; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10191; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10192; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10193; GFX7-NEXT:    s_add_u32 s0, s0, 16
10194; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10195; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10196; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10197; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10198; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10199; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10200; GFX7-NEXT:    s_endpgm
10201;
10202; GFX10-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
10203; GFX10-WGP:       ; %bb.0: ; %entry
10204; GFX10-WGP-NEXT:    s_clause 0x1
10205; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10206; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10207; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10208; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10209; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10210; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10211; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10212; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10213; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10214; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10215; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10216; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10217; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10218; GFX10-WGP-NEXT:    buffer_gl0_inv
10219; GFX10-WGP-NEXT:    s_endpgm
10220;
10221; GFX10-CU-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
10222; GFX10-CU:       ; %bb.0: ; %entry
10223; GFX10-CU-NEXT:    s_clause 0x1
10224; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10225; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10226; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10227; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10228; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10229; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10230; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10231; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10232; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10233; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10234; GFX10-CU-NEXT:    s_endpgm
10235;
10236; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
10237; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10238; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10239; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10240; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10241; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10242; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10243; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10244; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10245; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10246; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10247; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10248; SKIP-CACHE-INV-NEXT:    s_endpgm
10249;
10250; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
10251; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10252; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10253; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10254; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10255; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10256; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10257; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10258; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10259;
10260; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
10261; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10262; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10263; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10264; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10265; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10266; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10267; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10268; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10269; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10270; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10271; GFX90A-TGSPLIT-NEXT:    s_endpgm
10272;
10273; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
10274; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10275; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10276; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10277; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10278; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10279; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10280; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10281; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10282;
10283; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
10284; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10285; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10286; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10287; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10288; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10289; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10290; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10291; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10292; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10293; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
10294; GFX940-TGSPLIT-NEXT:    s_endpgm
10295;
10296; GFX11-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
10297; GFX11-WGP:       ; %bb.0: ; %entry
10298; GFX11-WGP-NEXT:    s_clause 0x1
10299; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10300; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10301; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10302; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10303; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10304; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10305; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10306; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10307; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10308; GFX11-WGP-NEXT:    buffer_gl0_inv
10309; GFX11-WGP-NEXT:    s_endpgm
10310;
10311; GFX11-CU-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
10312; GFX11-CU:       ; %bb.0: ; %entry
10313; GFX11-CU-NEXT:    s_clause 0x1
10314; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10315; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10316; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10317; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10318; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10319; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10320; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10321; GFX11-CU-NEXT:    s_endpgm
10322    i32* %out, i32 %in, i32 %old) {
10323entry:
10324  %gep = getelementptr i32, i32* %out, i32 4
10325  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release seq_cst
10326  ret void
10327}
10328
10329define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
10330; GFX7-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
10331; GFX7:       ; %bb.0: ; %entry
10332; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10333; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10334; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10335; GFX7-NEXT:    s_add_u32 s0, s0, 16
10336; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10337; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10338; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10339; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10340; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10341; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10342; GFX7-NEXT:    s_endpgm
10343;
10344; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
10345; GFX10-WGP:       ; %bb.0: ; %entry
10346; GFX10-WGP-NEXT:    s_clause 0x1
10347; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10348; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10349; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10350; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10351; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10352; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10353; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10354; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10355; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10356; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10357; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10358; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10359; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10360; GFX10-WGP-NEXT:    buffer_gl0_inv
10361; GFX10-WGP-NEXT:    s_endpgm
10362;
10363; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
10364; GFX10-CU:       ; %bb.0: ; %entry
10365; GFX10-CU-NEXT:    s_clause 0x1
10366; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10367; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10368; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10369; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10370; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10371; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10372; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10373; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10374; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10375; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10376; GFX10-CU-NEXT:    s_endpgm
10377;
10378; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
10379; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10380; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10381; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10382; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10383; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10384; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10385; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10386; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10387; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10388; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10389; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10390; SKIP-CACHE-INV-NEXT:    s_endpgm
10391;
10392; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
10393; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10394; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10395; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10396; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10397; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10398; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10399; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10400; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10401;
10402; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
10403; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10404; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10405; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10406; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10407; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10408; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10409; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10410; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10411; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10412; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10413; GFX90A-TGSPLIT-NEXT:    s_endpgm
10414;
10415; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
10416; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10417; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10418; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10419; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10420; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10421; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10422; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10423; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10424;
10425; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
10426; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10427; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10428; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10429; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10430; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10431; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10432; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10433; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10434; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10435; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
10436; GFX940-TGSPLIT-NEXT:    s_endpgm
10437;
10438; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
10439; GFX11-WGP:       ; %bb.0: ; %entry
10440; GFX11-WGP-NEXT:    s_clause 0x1
10441; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10442; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10443; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10444; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10445; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10446; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10447; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10448; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10449; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10450; GFX11-WGP-NEXT:    buffer_gl0_inv
10451; GFX11-WGP-NEXT:    s_endpgm
10452;
10453; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
10454; GFX11-CU:       ; %bb.0: ; %entry
10455; GFX11-CU-NEXT:    s_clause 0x1
10456; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10457; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10458; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10459; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10460; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10461; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10462; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10463; GFX11-CU-NEXT:    s_endpgm
10464    i32* %out, i32 %in, i32 %old) {
10465entry:
10466  %gep = getelementptr i32, i32* %out, i32 4
10467  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel seq_cst
10468  ret void
10469}
10470
10471define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
10472; GFX7-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
10473; GFX7:       ; %bb.0: ; %entry
10474; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10475; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10476; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10477; GFX7-NEXT:    s_add_u32 s0, s0, 16
10478; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10479; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10480; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10481; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10482; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10483; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10484; GFX7-NEXT:    s_endpgm
10485;
10486; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
10487; GFX10-WGP:       ; %bb.0: ; %entry
10488; GFX10-WGP-NEXT:    s_clause 0x1
10489; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10490; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10491; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10492; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10493; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10494; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10495; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10496; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10497; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10498; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10499; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10500; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10501; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10502; GFX10-WGP-NEXT:    buffer_gl0_inv
10503; GFX10-WGP-NEXT:    s_endpgm
10504;
10505; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
10506; GFX10-CU:       ; %bb.0: ; %entry
10507; GFX10-CU-NEXT:    s_clause 0x1
10508; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10509; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10510; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10511; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10512; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10513; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10514; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10515; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10516; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10517; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10518; GFX10-CU-NEXT:    s_endpgm
10519;
10520; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
10521; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10522; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10523; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10524; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10525; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10526; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10527; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10528; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10529; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10530; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10531; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10532; SKIP-CACHE-INV-NEXT:    s_endpgm
10533;
10534; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
10535; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10536; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10537; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10538; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10539; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10540; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10541; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10542; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10543;
10544; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
10545; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10546; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10547; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10548; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10549; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10550; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10551; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10552; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10553; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10554; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10555; GFX90A-TGSPLIT-NEXT:    s_endpgm
10556;
10557; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
10558; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10559; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10560; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10561; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10562; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10563; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10564; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10565; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10566;
10567; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
10568; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10569; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10570; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10571; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10572; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10573; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10574; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10575; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10576; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10577; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
10578; GFX940-TGSPLIT-NEXT:    s_endpgm
10579;
10580; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
10581; GFX11-WGP:       ; %bb.0: ; %entry
10582; GFX11-WGP-NEXT:    s_clause 0x1
10583; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10584; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10585; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10586; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10587; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10588; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10589; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10590; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10591; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10592; GFX11-WGP-NEXT:    buffer_gl0_inv
10593; GFX11-WGP-NEXT:    s_endpgm
10594;
10595; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
10596; GFX11-CU:       ; %bb.0: ; %entry
10597; GFX11-CU-NEXT:    s_clause 0x1
10598; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10599; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10600; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10601; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10602; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10603; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10604; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10605; GFX11-CU-NEXT:    s_endpgm
10606    i32* %out, i32 %in, i32 %old) {
10607entry:
10608  %gep = getelementptr i32, i32* %out, i32 4
10609  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
10610  ret void
10611}
10612
10613define amdgpu_kernel void @flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg(
10614; GFX7-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
10615; GFX7:       ; %bb.0: ; %entry
10616; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10617; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10618; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10619; GFX7-NEXT:    s_add_u32 s4, s0, 16
10620; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10621; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10622; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10623; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10624; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10625; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10626; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10627; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10628; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10629; GFX7-NEXT:    flat_store_dword v[0:1], v2
10630; GFX7-NEXT:    s_endpgm
10631;
10632; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
10633; GFX10-WGP:       ; %bb.0: ; %entry
10634; GFX10-WGP-NEXT:    s_clause 0x1
10635; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10636; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10637; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10638; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10639; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10640; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10641; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10642; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10643; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10644; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10645; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10646; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10647; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10648; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10649; GFX10-WGP-NEXT:    s_endpgm
10650;
10651; GFX10-CU-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
10652; GFX10-CU:       ; %bb.0: ; %entry
10653; GFX10-CU-NEXT:    s_clause 0x1
10654; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10655; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10656; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10657; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10658; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10659; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10660; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10661; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10662; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10663; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10664; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10665; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10666; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10667; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10668; GFX10-CU-NEXT:    s_endpgm
10669;
10670; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
10671; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10672; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10673; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10674; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10675; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10676; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10677; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10678; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10679; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10680; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10681; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10682; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10683; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10684; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10685; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10686; SKIP-CACHE-INV-NEXT:    s_endpgm
10687;
10688; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
10689; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10690; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10691; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10692; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10693; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10694; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10695; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10696; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10697; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10698; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10699;
10700; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
10701; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10702; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10703; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10704; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10705; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10706; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10707; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10708; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10709; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10710; GFX90A-TGSPLIT-NEXT:    s_endpgm
10711;
10712; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
10713; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10714; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10715; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10716; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10717; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10718; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10719; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10720; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10721; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10722; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10723;
10724; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
10725; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10726; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10727; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10728; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10729; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10730; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10731; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10732; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10733; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10734; GFX940-TGSPLIT-NEXT:    s_endpgm
10735;
10736; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
10737; GFX11-WGP:       ; %bb.0: ; %entry
10738; GFX11-WGP-NEXT:    s_clause 0x1
10739; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10740; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10741; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10742; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10743; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10744; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10745; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10746; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
10747; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10748; GFX11-WGP-NEXT:    s_endpgm
10749;
10750; GFX11-CU-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
10751; GFX11-CU:       ; %bb.0: ; %entry
10752; GFX11-CU-NEXT:    s_clause 0x1
10753; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10754; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10755; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10756; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10757; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10758; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10759; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10760; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
10761; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10762; GFX11-CU-NEXT:    s_endpgm
10763    i32* %out, i32 %in, i32 %old) {
10764entry:
10765  %gep = getelementptr i32, i32* %out, i32 4
10766  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic
10767  %val0 = extractvalue { i32, i1 } %val, 0
10768  store i32 %val0, i32* %out, align 4
10769  ret void
10770}
10771
10772define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
10773; GFX7-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
10774; GFX7:       ; %bb.0: ; %entry
10775; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10776; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10777; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10778; GFX7-NEXT:    s_add_u32 s4, s0, 16
10779; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10780; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10781; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10782; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10783; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10784; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10785; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10786; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10787; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10788; GFX7-NEXT:    flat_store_dword v[0:1], v2
10789; GFX7-NEXT:    s_endpgm
10790;
10791; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
10792; GFX10-WGP:       ; %bb.0: ; %entry
10793; GFX10-WGP-NEXT:    s_clause 0x1
10794; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10795; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10796; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10797; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10798; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10799; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10800; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10801; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10802; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10803; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10804; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10805; GFX10-WGP-NEXT:    buffer_gl0_inv
10806; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10807; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10808; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10809; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10810; GFX10-WGP-NEXT:    s_endpgm
10811;
10812; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
10813; GFX10-CU:       ; %bb.0: ; %entry
10814; GFX10-CU-NEXT:    s_clause 0x1
10815; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10816; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10817; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10818; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10819; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10820; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10821; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10822; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10823; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10824; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10825; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10826; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10827; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10828; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10829; GFX10-CU-NEXT:    s_endpgm
10830;
10831; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
10832; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10833; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10834; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10835; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10836; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10837; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10838; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10839; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10840; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10841; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10842; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10843; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10844; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10845; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10846; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10847; SKIP-CACHE-INV-NEXT:    s_endpgm
10848;
10849; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
10850; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10851; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10852; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10853; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10854; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10855; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10856; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10857; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10858; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10859; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10860;
10861; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
10862; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10863; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10864; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10865; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10866; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10867; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10868; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10869; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10870; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10871; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10872; GFX90A-TGSPLIT-NEXT:    s_endpgm
10873;
10874; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
10875; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10876; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10877; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10878; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10879; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10880; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10881; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10882; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10883; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10884; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10885;
10886; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
10887; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10888; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10889; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10890; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10891; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10892; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10893; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10894; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10895; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
10896; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10897; GFX940-TGSPLIT-NEXT:    s_endpgm
10898;
10899; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
10900; GFX11-WGP:       ; %bb.0: ; %entry
10901; GFX11-WGP-NEXT:    s_clause 0x1
10902; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10903; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10904; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10905; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10906; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10907; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10908; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10909; GFX11-WGP-NEXT:    buffer_gl0_inv
10910; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10911; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
10912; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10913; GFX11-WGP-NEXT:    s_endpgm
10914;
10915; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
10916; GFX11-CU:       ; %bb.0: ; %entry
10917; GFX11-CU-NEXT:    s_clause 0x1
10918; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10919; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10920; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10921; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10922; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10923; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10924; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10925; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
10926; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10927; GFX11-CU-NEXT:    s_endpgm
10928    i32* %out, i32 %in, i32 %old) {
10929entry:
10930  %gep = getelementptr i32, i32* %out, i32 4
10931  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
10932  %val0 = extractvalue { i32, i1 } %val, 0
10933  store i32 %val0, i32* %out, align 4
10934  ret void
10935}
10936
10937define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_ret_cmpxchg(
10938; GFX7-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
10939; GFX7:       ; %bb.0: ; %entry
10940; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10941; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10942; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10943; GFX7-NEXT:    s_add_u32 s4, s0, 16
10944; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10945; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10946; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10947; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10948; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10949; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10950; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10951; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10952; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10953; GFX7-NEXT:    flat_store_dword v[0:1], v2
10954; GFX7-NEXT:    s_endpgm
10955;
10956; GFX10-WGP-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
10957; GFX10-WGP:       ; %bb.0: ; %entry
10958; GFX10-WGP-NEXT:    s_clause 0x1
10959; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10960; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10961; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10962; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10963; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10964; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10965; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10966; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10967; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10968; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10969; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10970; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10971; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10972; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10973; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10974; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10975; GFX10-WGP-NEXT:    s_endpgm
10976;
10977; GFX10-CU-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
10978; GFX10-CU:       ; %bb.0: ; %entry
10979; GFX10-CU-NEXT:    s_clause 0x1
10980; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10981; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10982; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10983; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10984; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10985; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10986; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10987; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10988; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10989; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10990; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10991; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10992; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10993; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10994; GFX10-CU-NEXT:    s_endpgm
10995;
10996; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
10997; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10998; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10999; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11000; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11001; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11002; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11003; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11004; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11005; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11006; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11007; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11008; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11009; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11010; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11011; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11012; SKIP-CACHE-INV-NEXT:    s_endpgm
11013;
11014; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
11015; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11016; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11017; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11018; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11019; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11020; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11021; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11022; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11023; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11024; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11025;
11026; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
11027; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11028; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11029; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11030; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11031; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11032; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11033; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11034; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11035; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11036; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11037; GFX90A-TGSPLIT-NEXT:    s_endpgm
11038;
11039; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
11040; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11041; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11042; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11043; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11044; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11045; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11046; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11047; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11048; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11049; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11050;
11051; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
11052; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11053; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11054; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11055; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11056; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11057; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11058; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11059; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11060; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11061; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11062; GFX940-TGSPLIT-NEXT:    s_endpgm
11063;
11064; GFX11-WGP-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
11065; GFX11-WGP:       ; %bb.0: ; %entry
11066; GFX11-WGP-NEXT:    s_clause 0x1
11067; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11068; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11069; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11070; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11071; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11072; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11073; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11074; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11075; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11076; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11077; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11078; GFX11-WGP-NEXT:    s_endpgm
11079;
11080; GFX11-CU-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
11081; GFX11-CU:       ; %bb.0: ; %entry
11082; GFX11-CU-NEXT:    s_clause 0x1
11083; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11084; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11085; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11086; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11087; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11088; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11089; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11090; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11091; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11092; GFX11-CU-NEXT:    s_endpgm
11093    i32* %out, i32 %in, i32 %old) {
11094entry:
11095  %gep = getelementptr i32, i32* %out, i32 4
11096  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic
11097  %val0 = extractvalue { i32, i1 } %val, 0
11098  store i32 %val0, i32* %out, align 4
11099  ret void
11100}
11101
11102define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
11103; GFX7-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
11104; GFX7:       ; %bb.0: ; %entry
11105; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11106; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11107; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11108; GFX7-NEXT:    s_add_u32 s4, s0, 16
11109; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11110; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11111; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11112; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11113; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11114; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11115; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11116; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11117; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11118; GFX7-NEXT:    flat_store_dword v[0:1], v2
11119; GFX7-NEXT:    s_endpgm
11120;
11121; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
11122; GFX10-WGP:       ; %bb.0: ; %entry
11123; GFX10-WGP-NEXT:    s_clause 0x1
11124; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11125; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11126; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11127; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11128; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11129; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11130; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11131; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11132; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11133; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11134; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11135; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11136; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11137; GFX10-WGP-NEXT:    buffer_gl0_inv
11138; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11139; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11140; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11141; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11142; GFX10-WGP-NEXT:    s_endpgm
11143;
11144; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
11145; GFX10-CU:       ; %bb.0: ; %entry
11146; GFX10-CU-NEXT:    s_clause 0x1
11147; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11148; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11149; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11150; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11151; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11152; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11153; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11154; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11155; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11156; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11157; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11158; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11159; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11160; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11161; GFX10-CU-NEXT:    s_endpgm
11162;
11163; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
11164; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11165; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11166; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11167; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11168; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11169; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11170; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11171; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11172; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11173; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11174; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11175; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11176; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11177; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11178; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11179; SKIP-CACHE-INV-NEXT:    s_endpgm
11180;
11181; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
11182; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11183; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11184; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11185; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11186; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11187; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11188; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11189; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11190; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11191; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11192;
11193; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
11194; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11195; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11196; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11197; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11198; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11199; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11200; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11201; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11202; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11203; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11204; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11205; GFX90A-TGSPLIT-NEXT:    s_endpgm
11206;
11207; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
11208; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11209; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11210; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11211; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11212; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11213; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11214; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11215; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11216; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11217; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11218;
11219; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
11220; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11221; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11222; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11223; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11224; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11225; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11226; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11227; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11228; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11229; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
11230; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11231; GFX940-TGSPLIT-NEXT:    s_endpgm
11232;
11233; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
11234; GFX11-WGP:       ; %bb.0: ; %entry
11235; GFX11-WGP-NEXT:    s_clause 0x1
11236; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11237; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11238; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11239; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11240; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11241; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11242; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11243; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11244; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11245; GFX11-WGP-NEXT:    buffer_gl0_inv
11246; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11247; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11248; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11249; GFX11-WGP-NEXT:    s_endpgm
11250;
11251; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
11252; GFX11-CU:       ; %bb.0: ; %entry
11253; GFX11-CU-NEXT:    s_clause 0x1
11254; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11255; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11256; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11257; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11258; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11259; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11260; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11261; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11262; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11263; GFX11-CU-NEXT:    s_endpgm
11264    i32* %out, i32 %in, i32 %old) {
11265entry:
11266  %gep = getelementptr i32, i32* %out, i32 4
11267  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
11268  %val0 = extractvalue { i32, i1 } %val, 0
11269  store i32 %val0, i32* %out, align 4
11270  ret void
11271}
11272
11273define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
11274; GFX7-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
11275; GFX7:       ; %bb.0: ; %entry
11276; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11277; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11278; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11279; GFX7-NEXT:    s_add_u32 s4, s0, 16
11280; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11281; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11282; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11283; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11284; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11285; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11286; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11287; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11288; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11289; GFX7-NEXT:    flat_store_dword v[0:1], v2
11290; GFX7-NEXT:    s_endpgm
11291;
11292; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
11293; GFX10-WGP:       ; %bb.0: ; %entry
11294; GFX10-WGP-NEXT:    s_clause 0x1
11295; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11296; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11297; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11298; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11299; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11300; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11301; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11302; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11303; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11304; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11305; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11306; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11307; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11308; GFX10-WGP-NEXT:    buffer_gl0_inv
11309; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11310; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11311; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11312; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11313; GFX10-WGP-NEXT:    s_endpgm
11314;
11315; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
11316; GFX10-CU:       ; %bb.0: ; %entry
11317; GFX10-CU-NEXT:    s_clause 0x1
11318; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11319; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11320; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11321; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11322; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11323; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11324; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11325; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11326; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11327; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11328; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11329; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11330; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11331; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11332; GFX10-CU-NEXT:    s_endpgm
11333;
11334; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
11335; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11336; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11337; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11338; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11339; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11340; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11341; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11342; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11343; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11344; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11345; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11346; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11347; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11348; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11349; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11350; SKIP-CACHE-INV-NEXT:    s_endpgm
11351;
11352; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
11353; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11354; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11355; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11356; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11357; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11358; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11359; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11360; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11361; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11362; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11363;
11364; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
11365; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11366; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11367; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11368; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11369; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11370; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11371; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11372; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11373; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11374; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11375; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11376; GFX90A-TGSPLIT-NEXT:    s_endpgm
11377;
11378; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
11379; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11380; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11381; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11382; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11383; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11384; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11385; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11386; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11387; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11388; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11389;
11390; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
11391; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11392; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11393; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11394; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11395; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11396; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11397; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11398; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11399; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11400; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
11401; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11402; GFX940-TGSPLIT-NEXT:    s_endpgm
11403;
11404; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
11405; GFX11-WGP:       ; %bb.0: ; %entry
11406; GFX11-WGP-NEXT:    s_clause 0x1
11407; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11408; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11409; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11410; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11411; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11412; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11413; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11414; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11415; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11416; GFX11-WGP-NEXT:    buffer_gl0_inv
11417; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11418; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11419; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11420; GFX11-WGP-NEXT:    s_endpgm
11421;
11422; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
11423; GFX11-CU:       ; %bb.0: ; %entry
11424; GFX11-CU-NEXT:    s_clause 0x1
11425; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11426; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11427; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11428; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11429; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11430; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11431; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11432; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11433; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11434; GFX11-CU-NEXT:    s_endpgm
11435    i32* %out, i32 %in, i32 %old) {
11436entry:
11437  %gep = getelementptr i32, i32* %out, i32 4
11438  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
11439  %val0 = extractvalue { i32, i1 } %val, 0
11440  store i32 %val0, i32* %out, align 4
11441  ret void
11442}
11443
11444define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg(
11445; GFX7-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
11446; GFX7:       ; %bb.0: ; %entry
11447; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11448; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11449; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11450; GFX7-NEXT:    s_add_u32 s4, s0, 16
11451; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11452; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11453; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11454; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11455; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11456; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11457; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11458; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11459; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11460; GFX7-NEXT:    flat_store_dword v[0:1], v2
11461; GFX7-NEXT:    s_endpgm
11462;
11463; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
11464; GFX10-WGP:       ; %bb.0: ; %entry
11465; GFX10-WGP-NEXT:    s_clause 0x1
11466; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11467; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11468; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11469; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11470; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11471; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11472; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11473; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11474; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11475; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11476; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11477; GFX10-WGP-NEXT:    buffer_gl0_inv
11478; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11479; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11480; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11481; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11482; GFX10-WGP-NEXT:    s_endpgm
11483;
11484; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
11485; GFX10-CU:       ; %bb.0: ; %entry
11486; GFX10-CU-NEXT:    s_clause 0x1
11487; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11488; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11489; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11490; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11491; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11492; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11493; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11494; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11495; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11496; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11497; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11498; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11499; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11500; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11501; GFX10-CU-NEXT:    s_endpgm
11502;
11503; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
11504; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11505; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11506; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11507; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11508; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11509; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11510; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11511; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11512; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11513; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11514; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11515; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11516; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11517; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11518; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11519; SKIP-CACHE-INV-NEXT:    s_endpgm
11520;
11521; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
11522; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11523; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11524; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11525; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11526; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11527; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11528; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11529; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11530; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11531; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11532;
11533; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
11534; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11535; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11536; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11537; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11538; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11539; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11540; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11541; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11542; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11543; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11544; GFX90A-TGSPLIT-NEXT:    s_endpgm
11545;
11546; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
11547; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11548; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11549; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11550; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11551; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11552; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11553; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11554; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11555; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11556; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11557;
11558; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
11559; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11560; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11561; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11562; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11563; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11564; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11565; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11566; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11567; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
11568; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11569; GFX940-TGSPLIT-NEXT:    s_endpgm
11570;
11571; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
11572; GFX11-WGP:       ; %bb.0: ; %entry
11573; GFX11-WGP-NEXT:    s_clause 0x1
11574; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11575; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11576; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11577; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11578; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11579; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11580; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11581; GFX11-WGP-NEXT:    buffer_gl0_inv
11582; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11583; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11584; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11585; GFX11-WGP-NEXT:    s_endpgm
11586;
11587; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
11588; GFX11-CU:       ; %bb.0: ; %entry
11589; GFX11-CU-NEXT:    s_clause 0x1
11590; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11591; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11592; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11593; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11594; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11595; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11596; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11597; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11598; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11599; GFX11-CU-NEXT:    s_endpgm
11600    i32* %out, i32 %in, i32 %old) {
11601entry:
11602  %gep = getelementptr i32, i32* %out, i32 4
11603  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic acquire
11604  %val0 = extractvalue { i32, i1 } %val, 0
11605  store i32 %val0, i32* %out, align 4
11606  ret void
11607}
11608
11609define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_ret_cmpxchg(
11610; GFX7-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
11611; GFX7:       ; %bb.0: ; %entry
11612; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11613; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11614; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11615; GFX7-NEXT:    s_add_u32 s4, s0, 16
11616; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11617; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11618; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11619; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11620; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11621; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11622; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11623; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11624; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11625; GFX7-NEXT:    flat_store_dword v[0:1], v2
11626; GFX7-NEXT:    s_endpgm
11627;
11628; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
11629; GFX10-WGP:       ; %bb.0: ; %entry
11630; GFX10-WGP-NEXT:    s_clause 0x1
11631; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11632; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11633; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11634; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11635; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11636; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11637; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11638; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11639; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11640; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11641; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11642; GFX10-WGP-NEXT:    buffer_gl0_inv
11643; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11644; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11645; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11646; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11647; GFX10-WGP-NEXT:    s_endpgm
11648;
11649; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
11650; GFX10-CU:       ; %bb.0: ; %entry
11651; GFX10-CU-NEXT:    s_clause 0x1
11652; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11653; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11654; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11655; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11656; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11657; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11658; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11659; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11660; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11661; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11662; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11663; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11664; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11665; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11666; GFX10-CU-NEXT:    s_endpgm
11667;
11668; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
11669; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11670; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11671; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11672; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11673; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11674; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11675; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11676; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11677; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11678; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11679; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11680; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11681; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11682; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11683; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11684; SKIP-CACHE-INV-NEXT:    s_endpgm
11685;
11686; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
11687; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11688; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11689; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11690; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11691; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11692; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11693; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11694; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11695; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11696; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11697;
11698; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
11699; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11700; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11701; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11702; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11703; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11704; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11705; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11706; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11707; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11708; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11709; GFX90A-TGSPLIT-NEXT:    s_endpgm
11710;
11711; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
11712; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11713; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11714; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11715; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11716; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11717; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11718; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11719; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11720; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11721; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11722;
11723; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
11724; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11725; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11726; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11727; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11728; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11729; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11730; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11731; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11732; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
11733; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11734; GFX940-TGSPLIT-NEXT:    s_endpgm
11735;
11736; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
11737; GFX11-WGP:       ; %bb.0: ; %entry
11738; GFX11-WGP-NEXT:    s_clause 0x1
11739; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11740; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11741; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11742; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11743; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11744; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11745; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11746; GFX11-WGP-NEXT:    buffer_gl0_inv
11747; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11748; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11749; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11750; GFX11-WGP-NEXT:    s_endpgm
11751;
11752; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
11753; GFX11-CU:       ; %bb.0: ; %entry
11754; GFX11-CU-NEXT:    s_clause 0x1
11755; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11756; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11757; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11758; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11759; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11760; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11761; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11762; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11763; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11764; GFX11-CU-NEXT:    s_endpgm
11765    i32* %out, i32 %in, i32 %old) {
11766entry:
11767  %gep = getelementptr i32, i32* %out, i32 4
11768  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
11769  %val0 = extractvalue { i32, i1 } %val, 0
11770  store i32 %val0, i32* %out, align 4
11771  ret void
11772}
11773
11774define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_ret_cmpxchg(
11775; GFX7-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
11776; GFX7:       ; %bb.0: ; %entry
11777; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11778; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11779; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11780; GFX7-NEXT:    s_add_u32 s4, s0, 16
11781; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11782; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11783; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11784; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11785; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11786; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11787; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11788; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11789; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11790; GFX7-NEXT:    flat_store_dword v[0:1], v2
11791; GFX7-NEXT:    s_endpgm
11792;
11793; GFX10-WGP-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
11794; GFX10-WGP:       ; %bb.0: ; %entry
11795; GFX10-WGP-NEXT:    s_clause 0x1
11796; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11797; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11798; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11799; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11800; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11801; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11802; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11803; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11804; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11805; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11806; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11807; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11808; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11809; GFX10-WGP-NEXT:    buffer_gl0_inv
11810; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11811; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11812; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11813; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11814; GFX10-WGP-NEXT:    s_endpgm
11815;
11816; GFX10-CU-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
11817; GFX10-CU:       ; %bb.0: ; %entry
11818; GFX10-CU-NEXT:    s_clause 0x1
11819; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11820; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11821; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11822; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11823; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11824; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11825; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11826; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11827; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11828; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11829; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11830; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11831; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11832; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11833; GFX10-CU-NEXT:    s_endpgm
11834;
11835; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
11836; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11837; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11838; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11839; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11840; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11841; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11842; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11843; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11844; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11845; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11846; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11847; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11848; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11849; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11850; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11851; SKIP-CACHE-INV-NEXT:    s_endpgm
11852;
11853; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
11854; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11855; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11856; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11857; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11858; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11859; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11860; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11861; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11862; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11863; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11864;
11865; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
11866; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11867; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11868; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11869; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11870; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11871; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11872; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11873; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11874; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11875; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11876; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11877; GFX90A-TGSPLIT-NEXT:    s_endpgm
11878;
11879; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
11880; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11881; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11882; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11883; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11884; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11885; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11886; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11887; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11888; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11889; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11890;
11891; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
11892; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11893; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11894; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11895; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11896; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11897; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11898; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11899; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11900; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11901; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
11902; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11903; GFX940-TGSPLIT-NEXT:    s_endpgm
11904;
11905; GFX11-WGP-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
11906; GFX11-WGP:       ; %bb.0: ; %entry
11907; GFX11-WGP-NEXT:    s_clause 0x1
11908; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11909; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11910; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11911; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11912; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11913; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11914; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11915; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11916; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11917; GFX11-WGP-NEXT:    buffer_gl0_inv
11918; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11919; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11920; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11921; GFX11-WGP-NEXT:    s_endpgm
11922;
11923; GFX11-CU-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
11924; GFX11-CU:       ; %bb.0: ; %entry
11925; GFX11-CU-NEXT:    s_clause 0x1
11926; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11927; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11928; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11929; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11930; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11931; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11932; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11933; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11934; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11935; GFX11-CU-NEXT:    s_endpgm
11936    i32* %out, i32 %in, i32 %old) {
11937entry:
11938  %gep = getelementptr i32, i32* %out, i32 4
11939  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
11940  %val0 = extractvalue { i32, i1 } %val, 0
11941  store i32 %val0, i32* %out, align 4
11942  ret void
11943}
11944
11945define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
11946; GFX7-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
11947; GFX7:       ; %bb.0: ; %entry
11948; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11949; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11950; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11951; GFX7-NEXT:    s_add_u32 s4, s0, 16
11952; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11953; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11954; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11955; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11956; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11957; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11958; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11959; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11960; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11961; GFX7-NEXT:    flat_store_dword v[0:1], v2
11962; GFX7-NEXT:    s_endpgm
11963;
11964; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
11965; GFX10-WGP:       ; %bb.0: ; %entry
11966; GFX10-WGP-NEXT:    s_clause 0x1
11967; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11968; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11969; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11970; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11971; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11972; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11973; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11974; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11975; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11976; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11977; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11978; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11979; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11980; GFX10-WGP-NEXT:    buffer_gl0_inv
11981; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11982; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11983; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11984; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11985; GFX10-WGP-NEXT:    s_endpgm
11986;
11987; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
11988; GFX10-CU:       ; %bb.0: ; %entry
11989; GFX10-CU-NEXT:    s_clause 0x1
11990; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11991; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11992; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11993; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11994; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11995; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11996; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11997; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11998; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11999; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12000; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12001; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12002; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12003; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12004; GFX10-CU-NEXT:    s_endpgm
12005;
12006; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
12007; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12008; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12009; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12010; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12011; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12012; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12013; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12014; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12015; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12016; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12017; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12018; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12019; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12020; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12021; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12022; SKIP-CACHE-INV-NEXT:    s_endpgm
12023;
12024; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
12025; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12026; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12027; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12028; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12029; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12030; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12031; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12032; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12033; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12034; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12035;
12036; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
12037; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12038; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12039; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12040; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12041; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12042; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12043; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12044; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12045; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12046; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12047; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12048; GFX90A-TGSPLIT-NEXT:    s_endpgm
12049;
12050; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
12051; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12052; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12053; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12054; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12055; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12056; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12057; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12058; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12059; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12060; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12061;
12062; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
12063; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12064; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12065; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12066; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12067; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12068; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12069; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12070; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12071; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12072; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
12073; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12074; GFX940-TGSPLIT-NEXT:    s_endpgm
12075;
12076; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
12077; GFX11-WGP:       ; %bb.0: ; %entry
12078; GFX11-WGP-NEXT:    s_clause 0x1
12079; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12080; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12081; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12082; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12083; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12084; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12085; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12086; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12087; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12088; GFX11-WGP-NEXT:    buffer_gl0_inv
12089; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12090; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12091; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12092; GFX11-WGP-NEXT:    s_endpgm
12093;
12094; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
12095; GFX11-CU:       ; %bb.0: ; %entry
12096; GFX11-CU-NEXT:    s_clause 0x1
12097; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12098; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12099; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12100; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12101; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12102; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12103; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12104; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12105; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12106; GFX11-CU-NEXT:    s_endpgm
12107    i32* %out, i32 %in, i32 %old) {
12108entry:
12109  %gep = getelementptr i32, i32* %out, i32 4
12110  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
12111  %val0 = extractvalue { i32, i1 } %val, 0
12112  store i32 %val0, i32* %out, align 4
12113  ret void
12114}
12115
12116define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
12117; GFX7-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
12118; GFX7:       ; %bb.0: ; %entry
12119; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12120; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12121; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12122; GFX7-NEXT:    s_add_u32 s4, s0, 16
12123; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12124; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12125; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12126; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12127; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12128; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12129; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12130; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12131; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12132; GFX7-NEXT:    flat_store_dword v[0:1], v2
12133; GFX7-NEXT:    s_endpgm
12134;
12135; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
12136; GFX10-WGP:       ; %bb.0: ; %entry
12137; GFX10-WGP-NEXT:    s_clause 0x1
12138; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12139; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12140; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12141; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12142; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12143; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12144; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12145; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12146; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12147; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12148; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12149; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12150; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12151; GFX10-WGP-NEXT:    buffer_gl0_inv
12152; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12153; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12154; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12155; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12156; GFX10-WGP-NEXT:    s_endpgm
12157;
12158; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
12159; GFX10-CU:       ; %bb.0: ; %entry
12160; GFX10-CU-NEXT:    s_clause 0x1
12161; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12162; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12163; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12164; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12165; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12166; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12167; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12168; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12169; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12170; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12171; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12172; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12173; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12174; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12175; GFX10-CU-NEXT:    s_endpgm
12176;
12177; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
12178; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12179; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12180; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12181; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12182; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12183; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12184; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12185; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12186; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12187; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12188; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12189; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12190; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12191; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12192; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12193; SKIP-CACHE-INV-NEXT:    s_endpgm
12194;
12195; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
12196; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12197; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12198; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12199; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12200; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12201; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12202; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12203; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12204; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12205; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12206;
12207; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
12208; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12209; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12210; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12211; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12212; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12213; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12214; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12215; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12216; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12217; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12218; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12219; GFX90A-TGSPLIT-NEXT:    s_endpgm
12220;
12221; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
12222; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12223; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12224; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12225; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12226; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12227; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12228; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12229; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12230; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12231; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12232;
12233; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
12234; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12235; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12236; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12237; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12238; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12239; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12240; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12241; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12242; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12243; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
12244; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12245; GFX940-TGSPLIT-NEXT:    s_endpgm
12246;
12247; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
12248; GFX11-WGP:       ; %bb.0: ; %entry
12249; GFX11-WGP-NEXT:    s_clause 0x1
12250; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12251; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12252; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12253; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12254; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12255; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12256; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12257; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12258; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12259; GFX11-WGP-NEXT:    buffer_gl0_inv
12260; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12261; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12262; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12263; GFX11-WGP-NEXT:    s_endpgm
12264;
12265; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
12266; GFX11-CU:       ; %bb.0: ; %entry
12267; GFX11-CU-NEXT:    s_clause 0x1
12268; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12269; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12270; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12271; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12272; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12273; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12274; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12275; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12276; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12277; GFX11-CU-NEXT:    s_endpgm
12278    i32* %out, i32 %in, i32 %old) {
12279entry:
12280  %gep = getelementptr i32, i32* %out, i32 4
12281  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
12282  %val0 = extractvalue { i32, i1 } %val, 0
12283  store i32 %val0, i32* %out, align 4
12284  ret void
12285}
12286
12287define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg(
12288; GFX7-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
12289; GFX7:       ; %bb.0: ; %entry
12290; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12291; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12292; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12293; GFX7-NEXT:    s_add_u32 s4, s0, 16
12294; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12295; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12296; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12297; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12298; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12299; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12300; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12301; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12302; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12303; GFX7-NEXT:    flat_store_dword v[0:1], v2
12304; GFX7-NEXT:    s_endpgm
12305;
12306; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
12307; GFX10-WGP:       ; %bb.0: ; %entry
12308; GFX10-WGP-NEXT:    s_clause 0x1
12309; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12310; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12311; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12312; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12313; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12314; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12315; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12316; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12317; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12318; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12319; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12320; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12321; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12322; GFX10-WGP-NEXT:    buffer_gl0_inv
12323; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12324; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12325; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12326; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12327; GFX10-WGP-NEXT:    s_endpgm
12328;
12329; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
12330; GFX10-CU:       ; %bb.0: ; %entry
12331; GFX10-CU-NEXT:    s_clause 0x1
12332; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12333; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12334; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12335; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12336; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12337; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12338; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12339; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12340; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12341; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12342; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12343; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12344; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12345; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12346; GFX10-CU-NEXT:    s_endpgm
12347;
12348; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
12349; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12350; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12351; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12352; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12353; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12354; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12355; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12356; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12357; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12358; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12359; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12360; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12361; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12362; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12363; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12364; SKIP-CACHE-INV-NEXT:    s_endpgm
12365;
12366; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
12367; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12368; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12369; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12370; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12371; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12372; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12373; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12374; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12375; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12376; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12377;
12378; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
12379; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12380; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12381; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12382; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12383; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12384; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12385; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12386; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12387; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12388; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12389; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12390; GFX90A-TGSPLIT-NEXT:    s_endpgm
12391;
12392; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
12393; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12394; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12395; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12396; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12397; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12398; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12399; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12400; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12401; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12402; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12403;
12404; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
12405; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12406; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12407; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12408; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12409; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12410; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12411; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12412; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12413; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12414; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
12415; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12416; GFX940-TGSPLIT-NEXT:    s_endpgm
12417;
12418; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
12419; GFX11-WGP:       ; %bb.0: ; %entry
12420; GFX11-WGP-NEXT:    s_clause 0x1
12421; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12422; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12423; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12424; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12425; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12426; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12427; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12428; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12429; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12430; GFX11-WGP-NEXT:    buffer_gl0_inv
12431; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12432; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12433; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12434; GFX11-WGP-NEXT:    s_endpgm
12435;
12436; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
12437; GFX11-CU:       ; %bb.0: ; %entry
12438; GFX11-CU-NEXT:    s_clause 0x1
12439; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12440; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12441; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12442; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12443; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12444; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12445; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12446; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12447; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12448; GFX11-CU-NEXT:    s_endpgm
12449    i32* %out, i32 %in, i32 %old) {
12450entry:
12451  %gep = getelementptr i32, i32* %out, i32 4
12452  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic seq_cst
12453  %val0 = extractvalue { i32, i1 } %val, 0
12454  store i32 %val0, i32* %out, align 4
12455  ret void
12456}
12457
12458define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
12459; GFX7-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
12460; GFX7:       ; %bb.0: ; %entry
12461; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12462; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12463; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12464; GFX7-NEXT:    s_add_u32 s4, s0, 16
12465; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12466; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12467; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12468; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12469; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12470; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12471; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12472; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12473; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12474; GFX7-NEXT:    flat_store_dword v[0:1], v2
12475; GFX7-NEXT:    s_endpgm
12476;
12477; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
12478; GFX10-WGP:       ; %bb.0: ; %entry
12479; GFX10-WGP-NEXT:    s_clause 0x1
12480; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12481; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12482; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12483; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12484; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12485; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12486; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12487; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12488; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12489; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12490; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12491; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12492; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12493; GFX10-WGP-NEXT:    buffer_gl0_inv
12494; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12495; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12496; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12497; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12498; GFX10-WGP-NEXT:    s_endpgm
12499;
12500; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
12501; GFX10-CU:       ; %bb.0: ; %entry
12502; GFX10-CU-NEXT:    s_clause 0x1
12503; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12504; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12505; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12506; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12507; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12508; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12509; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12510; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12511; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12512; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12513; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12514; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12515; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12516; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12517; GFX10-CU-NEXT:    s_endpgm
12518;
12519; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
12520; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12521; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12522; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12523; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12524; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12525; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12526; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12527; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12528; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12529; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12530; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12531; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12532; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12533; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12534; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12535; SKIP-CACHE-INV-NEXT:    s_endpgm
12536;
12537; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
12538; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12539; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12540; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12541; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12542; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12543; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12544; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12545; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12546; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12547; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12548;
12549; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
12550; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12551; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12552; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12553; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12554; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12555; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12556; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12557; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12558; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12559; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12560; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12561; GFX90A-TGSPLIT-NEXT:    s_endpgm
12562;
12563; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
12564; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12565; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12566; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12567; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12568; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12569; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12570; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12571; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12572; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12573; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12574;
12575; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
12576; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12577; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12578; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12579; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12580; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12581; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12582; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12583; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12584; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12585; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
12586; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12587; GFX940-TGSPLIT-NEXT:    s_endpgm
12588;
12589; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
12590; GFX11-WGP:       ; %bb.0: ; %entry
12591; GFX11-WGP-NEXT:    s_clause 0x1
12592; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12593; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12594; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12595; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12596; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12597; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12598; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12599; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12600; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12601; GFX11-WGP-NEXT:    buffer_gl0_inv
12602; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12603; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12604; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12605; GFX11-WGP-NEXT:    s_endpgm
12606;
12607; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
12608; GFX11-CU:       ; %bb.0: ; %entry
12609; GFX11-CU-NEXT:    s_clause 0x1
12610; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12611; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12612; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12613; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12614; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12615; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12616; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12617; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12618; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12619; GFX11-CU-NEXT:    s_endpgm
12620    i32* %out, i32 %in, i32 %old) {
12621entry:
12622  %gep = getelementptr i32, i32* %out, i32 4
12623  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire seq_cst
12624  %val0 = extractvalue { i32, i1 } %val, 0
12625  store i32 %val0, i32* %out, align 4
12626  ret void
12627}
12628
12629define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_ret_cmpxchg(
12630; GFX7-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
12631; GFX7:       ; %bb.0: ; %entry
12632; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12633; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12634; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12635; GFX7-NEXT:    s_add_u32 s4, s0, 16
12636; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12637; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12638; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12639; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12640; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12641; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12642; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12643; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12644; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12645; GFX7-NEXT:    flat_store_dword v[0:1], v2
12646; GFX7-NEXT:    s_endpgm
12647;
12648; GFX10-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
12649; GFX10-WGP:       ; %bb.0: ; %entry
12650; GFX10-WGP-NEXT:    s_clause 0x1
12651; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12652; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12653; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12654; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12655; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12656; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12657; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12658; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12659; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12660; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12661; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12662; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12663; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12664; GFX10-WGP-NEXT:    buffer_gl0_inv
12665; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12666; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12667; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12668; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12669; GFX10-WGP-NEXT:    s_endpgm
12670;
12671; GFX10-CU-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
12672; GFX10-CU:       ; %bb.0: ; %entry
12673; GFX10-CU-NEXT:    s_clause 0x1
12674; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12675; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12676; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12677; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12678; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12679; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12680; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12681; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12682; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12683; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12684; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12685; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12686; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12687; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12688; GFX10-CU-NEXT:    s_endpgm
12689;
12690; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
12691; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12692; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12693; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12694; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12695; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12696; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12697; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12698; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12699; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12700; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12701; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12702; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12703; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12704; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12705; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12706; SKIP-CACHE-INV-NEXT:    s_endpgm
12707;
12708; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
12709; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12710; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12711; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12712; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12713; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12714; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12715; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12716; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12717; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12718; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12719;
12720; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
12721; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12722; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12723; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12724; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12725; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12726; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12727; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12728; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12729; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12730; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12731; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12732; GFX90A-TGSPLIT-NEXT:    s_endpgm
12733;
12734; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
12735; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12736; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12737; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12738; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12739; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12740; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12741; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12742; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12743; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12744; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12745;
12746; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
12747; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12748; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12749; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12750; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12751; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12752; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12753; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12754; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12755; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12756; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
12757; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12758; GFX940-TGSPLIT-NEXT:    s_endpgm
12759;
12760; GFX11-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
12761; GFX11-WGP:       ; %bb.0: ; %entry
12762; GFX11-WGP-NEXT:    s_clause 0x1
12763; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12764; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12765; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12766; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12767; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12768; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12769; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12770; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12771; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12772; GFX11-WGP-NEXT:    buffer_gl0_inv
12773; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12774; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12775; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12776; GFX11-WGP-NEXT:    s_endpgm
12777;
12778; GFX11-CU-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
12779; GFX11-CU:       ; %bb.0: ; %entry
12780; GFX11-CU-NEXT:    s_clause 0x1
12781; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12782; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12783; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12784; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12785; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12786; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12787; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12788; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12789; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12790; GFX11-CU-NEXT:    s_endpgm
12791    i32* %out, i32 %in, i32 %old) {
12792entry:
12793  %gep = getelementptr i32, i32* %out, i32 4
12794  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release seq_cst
12795  %val0 = extractvalue { i32, i1 } %val, 0
12796  store i32 %val0, i32* %out, align 4
12797  ret void
12798}
12799
12800define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
12801; GFX7-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
12802; GFX7:       ; %bb.0: ; %entry
12803; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12804; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12805; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12806; GFX7-NEXT:    s_add_u32 s4, s0, 16
12807; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12808; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12809; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12810; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12811; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12812; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12813; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12814; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12815; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12816; GFX7-NEXT:    flat_store_dword v[0:1], v2
12817; GFX7-NEXT:    s_endpgm
12818;
12819; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
12820; GFX10-WGP:       ; %bb.0: ; %entry
12821; GFX10-WGP-NEXT:    s_clause 0x1
12822; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12823; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12824; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12825; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12826; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12827; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12828; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12829; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12830; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12831; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12832; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12833; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12834; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12835; GFX10-WGP-NEXT:    buffer_gl0_inv
12836; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12837; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12838; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12839; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12840; GFX10-WGP-NEXT:    s_endpgm
12841;
12842; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
12843; GFX10-CU:       ; %bb.0: ; %entry
12844; GFX10-CU-NEXT:    s_clause 0x1
12845; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12846; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12847; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12848; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12849; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12850; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12851; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12852; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12853; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12854; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12855; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12856; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12857; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12858; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12859; GFX10-CU-NEXT:    s_endpgm
12860;
12861; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
12862; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12863; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12864; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12865; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12866; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12867; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12868; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12869; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12870; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12871; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12872; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12873; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12874; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12875; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12876; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12877; SKIP-CACHE-INV-NEXT:    s_endpgm
12878;
12879; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
12880; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12881; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12882; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12883; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12884; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12885; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12886; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12887; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12888; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12889; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12890;
12891; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
12892; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12893; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12894; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12895; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12896; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12897; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12898; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12899; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12900; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12901; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12902; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12903; GFX90A-TGSPLIT-NEXT:    s_endpgm
12904;
12905; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
12906; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12907; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12908; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12909; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12910; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12911; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12912; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12913; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12914; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12915; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12916;
12917; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
12918; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12919; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12920; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12921; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12922; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12923; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12924; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12925; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12926; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12927; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
12928; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12929; GFX940-TGSPLIT-NEXT:    s_endpgm
12930;
12931; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
12932; GFX11-WGP:       ; %bb.0: ; %entry
12933; GFX11-WGP-NEXT:    s_clause 0x1
12934; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12935; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12936; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12937; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12938; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12939; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12940; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12941; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12942; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12943; GFX11-WGP-NEXT:    buffer_gl0_inv
12944; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12945; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12946; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12947; GFX11-WGP-NEXT:    s_endpgm
12948;
12949; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
12950; GFX11-CU:       ; %bb.0: ; %entry
12951; GFX11-CU-NEXT:    s_clause 0x1
12952; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12953; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12954; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12955; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12956; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12957; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12958; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12959; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12960; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12961; GFX11-CU-NEXT:    s_endpgm
12962    i32* %out, i32 %in, i32 %old) {
12963entry:
12964  %gep = getelementptr i32, i32* %out, i32 4
12965  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel seq_cst
12966  %val0 = extractvalue { i32, i1 } %val, 0
12967  store i32 %val0, i32* %out, align 4
12968  ret void
12969}
12970
12971define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
12972; GFX7-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
12973; GFX7:       ; %bb.0: ; %entry
12974; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12975; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12976; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12977; GFX7-NEXT:    s_add_u32 s4, s0, 16
12978; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12979; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12980; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12981; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12982; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12983; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12984; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12985; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12986; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12987; GFX7-NEXT:    flat_store_dword v[0:1], v2
12988; GFX7-NEXT:    s_endpgm
12989;
12990; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
12991; GFX10-WGP:       ; %bb.0: ; %entry
12992; GFX10-WGP-NEXT:    s_clause 0x1
12993; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12994; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12995; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12996; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12997; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12998; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12999; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13000; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13001; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13002; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13003; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13004; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13005; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13006; GFX10-WGP-NEXT:    buffer_gl0_inv
13007; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13008; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13009; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13010; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13011; GFX10-WGP-NEXT:    s_endpgm
13012;
13013; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
13014; GFX10-CU:       ; %bb.0: ; %entry
13015; GFX10-CU-NEXT:    s_clause 0x1
13016; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13017; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13018; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13019; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13020; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13021; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13022; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13023; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13024; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13025; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13026; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13027; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13028; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
13029; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13030; GFX10-CU-NEXT:    s_endpgm
13031;
13032; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
13033; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13034; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13035; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13036; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13037; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13038; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13039; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13040; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13041; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13042; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13043; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13044; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13045; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13046; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
13047; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13048; SKIP-CACHE-INV-NEXT:    s_endpgm
13049;
13050; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
13051; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13052; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13053; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13054; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13055; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13056; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13057; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13058; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
13059; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13060; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13061;
13062; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
13063; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13064; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13065; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13066; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13067; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13068; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13069; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13070; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13071; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13072; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13073; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13074; GFX90A-TGSPLIT-NEXT:    s_endpgm
13075;
13076; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
13077; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13078; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13079; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13080; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13081; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13082; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13083; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13084; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
13085; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13086; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13087;
13088; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
13089; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13090; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13091; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13092; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13093; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13094; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13095; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13096; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13097; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13098; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
13099; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13100; GFX940-TGSPLIT-NEXT:    s_endpgm
13101;
13102; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
13103; GFX11-WGP:       ; %bb.0: ; %entry
13104; GFX11-WGP-NEXT:    s_clause 0x1
13105; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13106; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13107; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13108; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13109; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13110; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13111; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13112; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13113; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13114; GFX11-WGP-NEXT:    buffer_gl0_inv
13115; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13116; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13117; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13118; GFX11-WGP-NEXT:    s_endpgm
13119;
13120; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
13121; GFX11-CU:       ; %bb.0: ; %entry
13122; GFX11-CU-NEXT:    s_clause 0x1
13123; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13124; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13125; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13126; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13127; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13128; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13129; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
13130; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13131; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13132; GFX11-CU-NEXT:    s_endpgm
13133    i32* %out, i32 %in, i32 %old) {
13134entry:
13135  %gep = getelementptr i32, i32* %out, i32 4
13136  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
13137  %val0 = extractvalue { i32, i1 } %val, 0
13138  store i32 %val0, i32* %out, align 4
13139  ret void
13140}
13141
13142