1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
8; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
9; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
10; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-WGP %s
11; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-CU %s
12
13define amdgpu_kernel void @flat_workgroup_unordered_load(
14; GFX7-LABEL: flat_workgroup_unordered_load:
15; GFX7:       ; %bb.0: ; %entry
16; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
17; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
18; GFX7-NEXT:    v_mov_b32_e32 v0, s0
19; GFX7-NEXT:    v_mov_b32_e32 v1, s1
20; GFX7-NEXT:    flat_load_dword v2, v[0:1]
21; GFX7-NEXT:    v_mov_b32_e32 v0, s2
22; GFX7-NEXT:    v_mov_b32_e32 v1, s3
23; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
24; GFX7-NEXT:    flat_store_dword v[0:1], v2
25; GFX7-NEXT:    s_endpgm
26;
27; GFX10-WGP-LABEL: flat_workgroup_unordered_load:
28; GFX10-WGP:       ; %bb.0: ; %entry
29; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
30; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
31; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
32; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
33; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
34; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
35; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
36; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
37; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
38; GFX10-WGP-NEXT:    s_endpgm
39;
40; GFX10-CU-LABEL: flat_workgroup_unordered_load:
41; GFX10-CU:       ; %bb.0: ; %entry
42; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
43; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
44; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
45; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
46; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
47; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
48; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
49; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
50; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
51; GFX10-CU-NEXT:    s_endpgm
52;
53; SKIP-CACHE-INV-LABEL: flat_workgroup_unordered_load:
54; SKIP-CACHE-INV:       ; %bb.0: ; %entry
55; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
56; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
57; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
58; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
59; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
60; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
61; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
62; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
63; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
64; SKIP-CACHE-INV-NEXT:    s_endpgm
65;
66; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_unordered_load:
67; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
68; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
69; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
70; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
71; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
72; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
73; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
74; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
75; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
76; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
77; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
78;
79; GFX90A-TGSPLIT-LABEL: flat_workgroup_unordered_load:
80; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
81; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
82; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
83; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
84; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
85; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
86; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
87; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
88; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
89; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
90; GFX90A-TGSPLIT-NEXT:    s_endpgm
91;
92; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_unordered_load:
93; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
94; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
95; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
96; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
97; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
98; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
99; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
100; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
101; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
102; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
103; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
104;
105; GFX940-TGSPLIT-LABEL: flat_workgroup_unordered_load:
106; GFX940-TGSPLIT:       ; %bb.0: ; %entry
107; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
108; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
109; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
110; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
111; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
112; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
113; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
114; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
115; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
116; GFX940-TGSPLIT-NEXT:    s_endpgm
117;
118; GFX11-WGP-LABEL: flat_workgroup_unordered_load:
119; GFX11-WGP:       ; %bb.0: ; %entry
120; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
121; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
122; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
123; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
124; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
125; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
126; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
127; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
128; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
129; GFX11-WGP-NEXT:    s_endpgm
130;
131; GFX11-CU-LABEL: flat_workgroup_unordered_load:
132; GFX11-CU:       ; %bb.0: ; %entry
133; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
134; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
135; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
136; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
137; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
138; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
139; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
140; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
141; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
142; GFX11-CU-NEXT:    s_endpgm
143    i32* %in, i32* %out) {
144entry:
145  %val = load atomic i32, i32* %in syncscope("workgroup") unordered, align 4
146  store i32 %val, i32* %out
147  ret void
148}
149
150define amdgpu_kernel void @flat_workgroup_monotonic_load(
151; GFX7-LABEL: flat_workgroup_monotonic_load:
152; GFX7:       ; %bb.0: ; %entry
153; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
154; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
155; GFX7-NEXT:    v_mov_b32_e32 v0, s0
156; GFX7-NEXT:    v_mov_b32_e32 v1, s1
157; GFX7-NEXT:    flat_load_dword v2, v[0:1]
158; GFX7-NEXT:    v_mov_b32_e32 v0, s2
159; GFX7-NEXT:    v_mov_b32_e32 v1, s3
160; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
161; GFX7-NEXT:    flat_store_dword v[0:1], v2
162; GFX7-NEXT:    s_endpgm
163;
164; GFX10-WGP-LABEL: flat_workgroup_monotonic_load:
165; GFX10-WGP:       ; %bb.0: ; %entry
166; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
167; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
168; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
169; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
170; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
171; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
172; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
173; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
174; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
175; GFX10-WGP-NEXT:    s_endpgm
176;
177; GFX10-CU-LABEL: flat_workgroup_monotonic_load:
178; GFX10-CU:       ; %bb.0: ; %entry
179; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
180; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
181; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
182; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
183; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
184; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
185; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
186; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
187; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
188; GFX10-CU-NEXT:    s_endpgm
189;
190; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_load:
191; SKIP-CACHE-INV:       ; %bb.0: ; %entry
192; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
193; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
194; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
195; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
196; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
197; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
198; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
199; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
200; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
201; SKIP-CACHE-INV-NEXT:    s_endpgm
202;
203; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_load:
204; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
205; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
206; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
207; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
208; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
209; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
210; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
211; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
212; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
213; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
214; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
215;
216; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_load:
217; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
218; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
219; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
220; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
221; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
222; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
223; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
224; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
225; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
226; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
227; GFX90A-TGSPLIT-NEXT:    s_endpgm
228;
229; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_load:
230; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
231; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
232; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
233; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
234; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
235; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
236; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
237; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
238; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
239; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
240; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
241;
242; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_load:
243; GFX940-TGSPLIT:       ; %bb.0: ; %entry
244; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
245; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
246; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
247; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
248; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
249; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
250; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
251; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
252; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
253; GFX940-TGSPLIT-NEXT:    s_endpgm
254;
255; GFX11-WGP-LABEL: flat_workgroup_monotonic_load:
256; GFX11-WGP:       ; %bb.0: ; %entry
257; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
258; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
259; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
260; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
261; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
262; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
263; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
264; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
265; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
266; GFX11-WGP-NEXT:    s_endpgm
267;
268; GFX11-CU-LABEL: flat_workgroup_monotonic_load:
269; GFX11-CU:       ; %bb.0: ; %entry
270; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
271; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
272; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
273; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
274; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
275; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
276; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
277; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
278; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
279; GFX11-CU-NEXT:    s_endpgm
280    i32* %in, i32* %out) {
281entry:
282  %val = load atomic i32, i32* %in syncscope("workgroup") monotonic, align 4
283  store i32 %val, i32* %out
284  ret void
285}
286
287define amdgpu_kernel void @flat_workgroup_acquire_load(
288; GFX7-LABEL: flat_workgroup_acquire_load:
289; GFX7:       ; %bb.0: ; %entry
290; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
291; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
292; GFX7-NEXT:    v_mov_b32_e32 v0, s0
293; GFX7-NEXT:    v_mov_b32_e32 v1, s1
294; GFX7-NEXT:    flat_load_dword v2, v[0:1]
295; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
296; GFX7-NEXT:    v_mov_b32_e32 v0, s2
297; GFX7-NEXT:    v_mov_b32_e32 v1, s3
298; GFX7-NEXT:    s_waitcnt vmcnt(0)
299; GFX7-NEXT:    flat_store_dword v[0:1], v2
300; GFX7-NEXT:    s_endpgm
301;
302; GFX10-WGP-LABEL: flat_workgroup_acquire_load:
303; GFX10-WGP:       ; %bb.0: ; %entry
304; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
305; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
306; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
307; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
308; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
309; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
310; GFX10-WGP-NEXT:    buffer_gl0_inv
311; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
312; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
313; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
314; GFX10-WGP-NEXT:    s_endpgm
315;
316; GFX10-CU-LABEL: flat_workgroup_acquire_load:
317; GFX10-CU:       ; %bb.0: ; %entry
318; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
319; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
320; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
321; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
322; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
323; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
324; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
325; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
326; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
327; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
328; GFX10-CU-NEXT:    s_endpgm
329;
330; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_load:
331; SKIP-CACHE-INV:       ; %bb.0: ; %entry
332; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
333; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
334; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
335; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
336; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
337; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
338; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
339; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
340; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
341; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
342; SKIP-CACHE-INV-NEXT:    s_endpgm
343;
344; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_load:
345; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
346; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
347; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
348; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
349; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
350; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
351; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
352; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
353; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
354; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
355; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
356; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
357;
358; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_load:
359; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
360; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
361; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
362; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
363; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
364; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
365; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
366; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
367; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
368; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
369; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
370; GFX90A-TGSPLIT-NEXT:    s_endpgm
371;
372; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_load:
373; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
374; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
375; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
376; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
377; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
378; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
379; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
380; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
381; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
382; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
383; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
384; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
385;
386; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_load:
387; GFX940-TGSPLIT:       ; %bb.0: ; %entry
388; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
389; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
390; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
391; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
392; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
393; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
394; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
395; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
396; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
397; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
398; GFX940-TGSPLIT-NEXT:    s_endpgm
399;
400; GFX11-WGP-LABEL: flat_workgroup_acquire_load:
401; GFX11-WGP:       ; %bb.0: ; %entry
402; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
403; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
404; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
405; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
406; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
407; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
408; GFX11-WGP-NEXT:    buffer_gl0_inv
409; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
410; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
411; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
412; GFX11-WGP-NEXT:    s_endpgm
413;
414; GFX11-CU-LABEL: flat_workgroup_acquire_load:
415; GFX11-CU:       ; %bb.0: ; %entry
416; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
417; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
418; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
419; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
420; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
421; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
422; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
423; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
424; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
425; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
426; GFX11-CU-NEXT:    s_endpgm
427    i32* %in, i32* %out) {
428entry:
429  %val = load atomic i32, i32* %in syncscope("workgroup") acquire, align 4
430  store i32 %val, i32* %out
431  ret void
432}
433
434define amdgpu_kernel void @flat_workgroup_seq_cst_load(
435; GFX7-LABEL: flat_workgroup_seq_cst_load:
436; GFX7:       ; %bb.0: ; %entry
437; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
438; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
439; GFX7-NEXT:    v_mov_b32_e32 v0, s0
440; GFX7-NEXT:    v_mov_b32_e32 v1, s1
441; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
442; GFX7-NEXT:    flat_load_dword v2, v[0:1]
443; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
444; GFX7-NEXT:    v_mov_b32_e32 v0, s2
445; GFX7-NEXT:    v_mov_b32_e32 v1, s3
446; GFX7-NEXT:    s_waitcnt vmcnt(0)
447; GFX7-NEXT:    flat_store_dword v[0:1], v2
448; GFX7-NEXT:    s_endpgm
449;
450; GFX10-WGP-LABEL: flat_workgroup_seq_cst_load:
451; GFX10-WGP:       ; %bb.0: ; %entry
452; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
453; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
454; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
455; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
456; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
457; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
458; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
459; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
460; GFX10-WGP-NEXT:    buffer_gl0_inv
461; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
462; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
463; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
464; GFX10-WGP-NEXT:    s_endpgm
465;
466; GFX10-CU-LABEL: flat_workgroup_seq_cst_load:
467; GFX10-CU:       ; %bb.0: ; %entry
468; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
469; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
470; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
471; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
472; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
473; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
474; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
475; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
476; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
477; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
478; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
479; GFX10-CU-NEXT:    s_endpgm
480;
481; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_load:
482; SKIP-CACHE-INV:       ; %bb.0: ; %entry
483; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
484; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
485; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
486; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
487; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
488; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
489; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
490; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
491; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
492; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
493; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
494; SKIP-CACHE-INV-NEXT:    s_endpgm
495;
496; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_load:
497; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
498; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
499; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
500; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
501; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
502; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
503; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
504; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
505; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
506; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
507; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
508; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
509; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
510;
511; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_load:
512; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
513; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
514; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
515; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
516; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
517; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
518; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
519; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
520; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
521; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
522; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
523; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
524; GFX90A-TGSPLIT-NEXT:    s_endpgm
525;
526; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_load:
527; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
528; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
529; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
530; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
531; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
532; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
533; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
534; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
535; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
536; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
537; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
538; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
539; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
540;
541; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_load:
542; GFX940-TGSPLIT:       ; %bb.0: ; %entry
543; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
544; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
545; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
546; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
547; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
548; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
549; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
550; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
551; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
552; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
553; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
554; GFX940-TGSPLIT-NEXT:    s_endpgm
555;
556; GFX11-WGP-LABEL: flat_workgroup_seq_cst_load:
557; GFX11-WGP:       ; %bb.0: ; %entry
558; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
559; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
560; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
561; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
562; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
563; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
564; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
565; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
566; GFX11-WGP-NEXT:    buffer_gl0_inv
567; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
568; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
569; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
570; GFX11-WGP-NEXT:    s_endpgm
571;
572; GFX11-CU-LABEL: flat_workgroup_seq_cst_load:
573; GFX11-CU:       ; %bb.0: ; %entry
574; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
575; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
576; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
577; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
578; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
579; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
580; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
581; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
582; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
583; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
584; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
585; GFX11-CU-NEXT:    s_endpgm
586    i32* %in, i32* %out) {
587entry:
588  %val = load atomic i32, i32* %in syncscope("workgroup") seq_cst, align 4
589  store i32 %val, i32* %out
590  ret void
591}
592
593define amdgpu_kernel void @flat_workgroup_unordered_store(
594; GFX7-LABEL: flat_workgroup_unordered_store:
595; GFX7:       ; %bb.0: ; %entry
596; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
597; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
598; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
599; GFX7-NEXT:    v_mov_b32_e32 v0, s0
600; GFX7-NEXT:    v_mov_b32_e32 v1, s1
601; GFX7-NEXT:    v_mov_b32_e32 v2, s2
602; GFX7-NEXT:    flat_store_dword v[0:1], v2
603; GFX7-NEXT:    s_endpgm
604;
605; GFX10-WGP-LABEL: flat_workgroup_unordered_store:
606; GFX10-WGP:       ; %bb.0: ; %entry
607; GFX10-WGP-NEXT:    s_clause 0x1
608; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
609; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
610; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
611; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
612; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
613; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
614; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
615; GFX10-WGP-NEXT:    s_endpgm
616;
617; GFX10-CU-LABEL: flat_workgroup_unordered_store:
618; GFX10-CU:       ; %bb.0: ; %entry
619; GFX10-CU-NEXT:    s_clause 0x1
620; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
621; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
622; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
623; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
624; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
625; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
626; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
627; GFX10-CU-NEXT:    s_endpgm
628;
629; SKIP-CACHE-INV-LABEL: flat_workgroup_unordered_store:
630; SKIP-CACHE-INV:       ; %bb.0: ; %entry
631; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
632; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
633; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
634; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
635; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
636; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
637; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
638; SKIP-CACHE-INV-NEXT:    s_endpgm
639;
640; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_unordered_store:
641; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
642; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
643; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
644; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
645; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
646; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
647; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
648; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
649;
650; GFX90A-TGSPLIT-LABEL: flat_workgroup_unordered_store:
651; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
652; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
653; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
654; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
655; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
656; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
657; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
658; GFX90A-TGSPLIT-NEXT:    s_endpgm
659;
660; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_unordered_store:
661; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
662; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
663; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
664; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
665; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
666; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
667; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
668; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
669;
670; GFX940-TGSPLIT-LABEL: flat_workgroup_unordered_store:
671; GFX940-TGSPLIT:       ; %bb.0: ; %entry
672; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
673; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
674; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
675; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
676; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
677; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
678; GFX940-TGSPLIT-NEXT:    s_endpgm
679;
680; GFX11-WGP-LABEL: flat_workgroup_unordered_store:
681; GFX11-WGP:       ; %bb.0: ; %entry
682; GFX11-WGP-NEXT:    s_clause 0x1
683; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
684; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
685; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
686; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
687; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
688; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
689; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
690; GFX11-WGP-NEXT:    s_endpgm
691;
692; GFX11-CU-LABEL: flat_workgroup_unordered_store:
693; GFX11-CU:       ; %bb.0: ; %entry
694; GFX11-CU-NEXT:    s_clause 0x1
695; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
696; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
697; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
698; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
699; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
700; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
701; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
702; GFX11-CU-NEXT:    s_endpgm
703    i32 %in, i32* %out) {
704entry:
705  store atomic i32 %in, i32* %out syncscope("workgroup") unordered, align 4
706  ret void
707}
708
709define amdgpu_kernel void @flat_workgroup_monotonic_store(
710; GFX7-LABEL: flat_workgroup_monotonic_store:
711; GFX7:       ; %bb.0: ; %entry
712; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
713; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
714; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
715; GFX7-NEXT:    v_mov_b32_e32 v0, s0
716; GFX7-NEXT:    v_mov_b32_e32 v1, s1
717; GFX7-NEXT:    v_mov_b32_e32 v2, s2
718; GFX7-NEXT:    flat_store_dword v[0:1], v2
719; GFX7-NEXT:    s_endpgm
720;
721; GFX10-WGP-LABEL: flat_workgroup_monotonic_store:
722; GFX10-WGP:       ; %bb.0: ; %entry
723; GFX10-WGP-NEXT:    s_clause 0x1
724; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
725; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
726; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
727; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
728; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
729; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
730; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
731; GFX10-WGP-NEXT:    s_endpgm
732;
733; GFX10-CU-LABEL: flat_workgroup_monotonic_store:
734; GFX10-CU:       ; %bb.0: ; %entry
735; GFX10-CU-NEXT:    s_clause 0x1
736; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
737; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
738; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
739; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
740; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
741; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
742; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
743; GFX10-CU-NEXT:    s_endpgm
744;
745; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_store:
746; SKIP-CACHE-INV:       ; %bb.0: ; %entry
747; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
748; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
749; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
750; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
751; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
752; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
753; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
754; SKIP-CACHE-INV-NEXT:    s_endpgm
755;
756; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_store:
757; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
758; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
759; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
760; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
761; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
762; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
763; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
764; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
765;
766; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_store:
767; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
768; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
769; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
770; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
771; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
772; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
773; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
774; GFX90A-TGSPLIT-NEXT:    s_endpgm
775;
776; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_store:
777; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
778; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
779; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
780; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
781; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
782; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
783; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
784; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
785;
786; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_store:
787; GFX940-TGSPLIT:       ; %bb.0: ; %entry
788; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
789; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
790; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
791; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
792; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
793; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
794; GFX940-TGSPLIT-NEXT:    s_endpgm
795;
796; GFX11-WGP-LABEL: flat_workgroup_monotonic_store:
797; GFX11-WGP:       ; %bb.0: ; %entry
798; GFX11-WGP-NEXT:    s_clause 0x1
799; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
800; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
801; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
802; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
803; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
804; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
805; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
806; GFX11-WGP-NEXT:    s_endpgm
807;
808; GFX11-CU-LABEL: flat_workgroup_monotonic_store:
809; GFX11-CU:       ; %bb.0: ; %entry
810; GFX11-CU-NEXT:    s_clause 0x1
811; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
812; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
813; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
814; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
815; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
816; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
817; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
818; GFX11-CU-NEXT:    s_endpgm
819    i32 %in, i32* %out) {
820entry:
821  store atomic i32 %in, i32* %out syncscope("workgroup") monotonic, align 4
822  ret void
823}
824
825define amdgpu_kernel void @flat_workgroup_release_store(
826; GFX7-LABEL: flat_workgroup_release_store:
827; GFX7:       ; %bb.0: ; %entry
828; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
829; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
830; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
831; GFX7-NEXT:    v_mov_b32_e32 v0, s0
832; GFX7-NEXT:    v_mov_b32_e32 v1, s1
833; GFX7-NEXT:    v_mov_b32_e32 v2, s2
834; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
835; GFX7-NEXT:    flat_store_dword v[0:1], v2
836; GFX7-NEXT:    s_endpgm
837;
838; GFX10-WGP-LABEL: flat_workgroup_release_store:
839; GFX10-WGP:       ; %bb.0: ; %entry
840; GFX10-WGP-NEXT:    s_clause 0x1
841; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
842; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
843; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
844; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
845; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
846; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
847; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
848; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
849; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
850; GFX10-WGP-NEXT:    s_endpgm
851;
852; GFX10-CU-LABEL: flat_workgroup_release_store:
853; GFX10-CU:       ; %bb.0: ; %entry
854; GFX10-CU-NEXT:    s_clause 0x1
855; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
856; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
857; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
858; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
859; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
860; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
861; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
862; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
863; GFX10-CU-NEXT:    s_endpgm
864;
865; SKIP-CACHE-INV-LABEL: flat_workgroup_release_store:
866; SKIP-CACHE-INV:       ; %bb.0: ; %entry
867; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
868; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
869; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
870; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
871; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
872; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
873; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
874; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
875; SKIP-CACHE-INV-NEXT:    s_endpgm
876;
877; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_store:
878; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
879; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
880; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
881; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
882; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
883; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
884; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
885; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
886; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
887;
888; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_store:
889; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
890; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
891; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
892; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
893; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
894; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
895; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
896; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
897; GFX90A-TGSPLIT-NEXT:    s_endpgm
898;
899; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_store:
900; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
901; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
902; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
903; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
904; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
905; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
906; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
907; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
908; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
909;
910; GFX940-TGSPLIT-LABEL: flat_workgroup_release_store:
911; GFX940-TGSPLIT:       ; %bb.0: ; %entry
912; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
913; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
914; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
915; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
916; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
917; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
918; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
919; GFX940-TGSPLIT-NEXT:    s_endpgm
920;
921; GFX11-WGP-LABEL: flat_workgroup_release_store:
922; GFX11-WGP:       ; %bb.0: ; %entry
923; GFX11-WGP-NEXT:    s_clause 0x1
924; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
925; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
926; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
927; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
928; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
929; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
930; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
931; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
932; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
933; GFX11-WGP-NEXT:    s_endpgm
934;
935; GFX11-CU-LABEL: flat_workgroup_release_store:
936; GFX11-CU:       ; %bb.0: ; %entry
937; GFX11-CU-NEXT:    s_clause 0x1
938; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
939; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
940; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
941; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
942; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
943; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
944; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
945; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
946; GFX11-CU-NEXT:    s_endpgm
947    i32 %in, i32* %out) {
948entry:
949  store atomic i32 %in, i32* %out syncscope("workgroup") release, align 4
950  ret void
951}
952
953define amdgpu_kernel void @flat_workgroup_seq_cst_store(
954; GFX7-LABEL: flat_workgroup_seq_cst_store:
955; GFX7:       ; %bb.0: ; %entry
956; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
957; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
958; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
959; GFX7-NEXT:    v_mov_b32_e32 v0, s0
960; GFX7-NEXT:    v_mov_b32_e32 v1, s1
961; GFX7-NEXT:    v_mov_b32_e32 v2, s2
962; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
963; GFX7-NEXT:    flat_store_dword v[0:1], v2
964; GFX7-NEXT:    s_endpgm
965;
966; GFX10-WGP-LABEL: flat_workgroup_seq_cst_store:
967; GFX10-WGP:       ; %bb.0: ; %entry
968; GFX10-WGP-NEXT:    s_clause 0x1
969; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
970; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
971; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
972; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
973; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
974; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
975; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
976; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
977; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
978; GFX10-WGP-NEXT:    s_endpgm
979;
980; GFX10-CU-LABEL: flat_workgroup_seq_cst_store:
981; GFX10-CU:       ; %bb.0: ; %entry
982; GFX10-CU-NEXT:    s_clause 0x1
983; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
984; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
985; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
986; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
987; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
988; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
989; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
990; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
991; GFX10-CU-NEXT:    s_endpgm
992;
993; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_store:
994; SKIP-CACHE-INV:       ; %bb.0: ; %entry
995; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
996; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
997; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
998; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
999; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1000; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1001; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1002; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1003; SKIP-CACHE-INV-NEXT:    s_endpgm
1004;
1005; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_store:
1006; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1007; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1008; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
1009; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1010; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1011; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1012; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1013; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1014; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1015;
1016; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_store:
1017; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1018; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1019; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
1020; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1021; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1022; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1023; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1024; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1025; GFX90A-TGSPLIT-NEXT:    s_endpgm
1026;
1027; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_store:
1028; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1029; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
1030; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
1031; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1032; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1033; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1034; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1035; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
1036; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1037;
1038; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_store:
1039; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1040; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
1041; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
1042; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1043; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1044; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1045; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1046; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
1047; GFX940-TGSPLIT-NEXT:    s_endpgm
1048;
1049; GFX11-WGP-LABEL: flat_workgroup_seq_cst_store:
1050; GFX11-WGP:       ; %bb.0: ; %entry
1051; GFX11-WGP-NEXT:    s_clause 0x1
1052; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
1053; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
1054; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1055; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1056; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1057; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1058; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1059; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1060; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
1061; GFX11-WGP-NEXT:    s_endpgm
1062;
1063; GFX11-CU-LABEL: flat_workgroup_seq_cst_store:
1064; GFX11-CU:       ; %bb.0: ; %entry
1065; GFX11-CU-NEXT:    s_clause 0x1
1066; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
1067; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
1068; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1069; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1070; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1071; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1072; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1073; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1074; GFX11-CU-NEXT:    s_endpgm
1075    i32 %in, i32* %out) {
1076entry:
1077  store atomic i32 %in, i32* %out syncscope("workgroup") seq_cst, align 4
1078  ret void
1079}
1080
1081define amdgpu_kernel void @flat_workgroup_monotonic_atomicrmw(
1082; GFX7-LABEL: flat_workgroup_monotonic_atomicrmw:
1083; GFX7:       ; %bb.0: ; %entry
1084; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1085; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1086; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1087; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1088; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1089; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1090; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1091; GFX7-NEXT:    s_endpgm
1092;
1093; GFX10-WGP-LABEL: flat_workgroup_monotonic_atomicrmw:
1094; GFX10-WGP:       ; %bb.0: ; %entry
1095; GFX10-WGP-NEXT:    s_clause 0x1
1096; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1097; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1098; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1099; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1100; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1101; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1102; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1103; GFX10-WGP-NEXT:    s_endpgm
1104;
1105; GFX10-CU-LABEL: flat_workgroup_monotonic_atomicrmw:
1106; GFX10-CU:       ; %bb.0: ; %entry
1107; GFX10-CU-NEXT:    s_clause 0x1
1108; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1109; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1110; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1111; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1112; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1113; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1114; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1115; GFX10-CU-NEXT:    s_endpgm
1116;
1117; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_atomicrmw:
1118; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1119; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1120; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1121; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1122; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1123; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1124; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1125; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1126; SKIP-CACHE-INV-NEXT:    s_endpgm
1127;
1128; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw:
1129; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1130; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1131; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1132; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1133; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1134; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1135; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1136; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1137;
1138; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw:
1139; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1140; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1141; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1142; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1143; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1144; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1145; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1146; GFX90A-TGSPLIT-NEXT:    s_endpgm
1147;
1148; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw:
1149; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1150; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1151; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1152; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1153; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1154; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1155; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1156; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1157;
1158; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw:
1159; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1160; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1161; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1162; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1163; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1164; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1165; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1166; GFX940-TGSPLIT-NEXT:    s_endpgm
1167;
1168; GFX11-WGP-LABEL: flat_workgroup_monotonic_atomicrmw:
1169; GFX11-WGP:       ; %bb.0: ; %entry
1170; GFX11-WGP-NEXT:    s_clause 0x1
1171; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1172; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1173; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1174; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1175; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1176; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1177; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1178; GFX11-WGP-NEXT:    s_endpgm
1179;
1180; GFX11-CU-LABEL: flat_workgroup_monotonic_atomicrmw:
1181; GFX11-CU:       ; %bb.0: ; %entry
1182; GFX11-CU-NEXT:    s_clause 0x1
1183; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1184; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1185; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1186; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1187; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1188; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1189; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1190; GFX11-CU-NEXT:    s_endpgm
1191    i32* %out, i32 %in) {
1192entry:
1193  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") monotonic
1194  ret void
1195}
1196
1197define amdgpu_kernel void @flat_workgroup_acquire_atomicrmw(
1198; GFX7-LABEL: flat_workgroup_acquire_atomicrmw:
1199; GFX7:       ; %bb.0: ; %entry
1200; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1201; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1202; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1203; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1204; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1205; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1206; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1207; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1208; GFX7-NEXT:    s_endpgm
1209;
1210; GFX10-WGP-LABEL: flat_workgroup_acquire_atomicrmw:
1211; GFX10-WGP:       ; %bb.0: ; %entry
1212; GFX10-WGP-NEXT:    s_clause 0x1
1213; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1214; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1215; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1216; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1217; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1218; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1219; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1220; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1221; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1222; GFX10-WGP-NEXT:    buffer_gl0_inv
1223; GFX10-WGP-NEXT:    s_endpgm
1224;
1225; GFX10-CU-LABEL: flat_workgroup_acquire_atomicrmw:
1226; GFX10-CU:       ; %bb.0: ; %entry
1227; GFX10-CU-NEXT:    s_clause 0x1
1228; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1229; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1230; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1231; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1232; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1233; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1234; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1235; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1236; GFX10-CU-NEXT:    s_endpgm
1237;
1238; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_atomicrmw:
1239; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1240; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1241; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1242; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1243; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1244; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1245; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1246; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1247; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1248; SKIP-CACHE-INV-NEXT:    s_endpgm
1249;
1250; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw:
1251; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1252; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1253; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1254; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1255; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1256; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1257; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1258; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1259; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1260;
1261; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw:
1262; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1263; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1264; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1265; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1266; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1267; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1268; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1269; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1270; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1271; GFX90A-TGSPLIT-NEXT:    s_endpgm
1272;
1273; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw:
1274; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1275; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1276; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1277; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1278; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1279; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1280; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1281; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1282; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1283;
1284; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw:
1285; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1286; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1287; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1288; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1289; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1290; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1291; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1292; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1293; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
1294; GFX940-TGSPLIT-NEXT:    s_endpgm
1295;
1296; GFX11-WGP-LABEL: flat_workgroup_acquire_atomicrmw:
1297; GFX11-WGP:       ; %bb.0: ; %entry
1298; GFX11-WGP-NEXT:    s_clause 0x1
1299; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1300; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1301; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1302; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1303; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1304; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1305; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1306; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1307; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1308; GFX11-WGP-NEXT:    buffer_gl0_inv
1309; GFX11-WGP-NEXT:    s_endpgm
1310;
1311; GFX11-CU-LABEL: flat_workgroup_acquire_atomicrmw:
1312; GFX11-CU:       ; %bb.0: ; %entry
1313; GFX11-CU-NEXT:    s_clause 0x1
1314; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1315; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1316; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1317; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1318; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1319; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1320; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1321; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1322; GFX11-CU-NEXT:    s_endpgm
1323    i32* %out, i32 %in) {
1324entry:
1325  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acquire
1326  ret void
1327}
1328
1329define amdgpu_kernel void @flat_workgroup_release_atomicrmw(
1330; GFX7-LABEL: flat_workgroup_release_atomicrmw:
1331; GFX7:       ; %bb.0: ; %entry
1332; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1333; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1334; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1335; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1336; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1337; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1338; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1339; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1340; GFX7-NEXT:    s_endpgm
1341;
1342; GFX10-WGP-LABEL: flat_workgroup_release_atomicrmw:
1343; GFX10-WGP:       ; %bb.0: ; %entry
1344; GFX10-WGP-NEXT:    s_clause 0x1
1345; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1346; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1347; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1348; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1349; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1350; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1351; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1352; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1353; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1354; GFX10-WGP-NEXT:    s_endpgm
1355;
1356; GFX10-CU-LABEL: flat_workgroup_release_atomicrmw:
1357; GFX10-CU:       ; %bb.0: ; %entry
1358; GFX10-CU-NEXT:    s_clause 0x1
1359; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1360; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1361; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1362; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1363; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1364; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1365; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1366; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1367; GFX10-CU-NEXT:    s_endpgm
1368;
1369; SKIP-CACHE-INV-LABEL: flat_workgroup_release_atomicrmw:
1370; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1371; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1372; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1373; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1374; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1375; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1376; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1377; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1378; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1379; SKIP-CACHE-INV-NEXT:    s_endpgm
1380;
1381; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_atomicrmw:
1382; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1383; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1384; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1385; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1386; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1387; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1388; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1389; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1390; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1391;
1392; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_atomicrmw:
1393; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1394; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1395; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1396; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1397; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1398; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1399; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1400; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1401; GFX90A-TGSPLIT-NEXT:    s_endpgm
1402;
1403; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_atomicrmw:
1404; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1405; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1406; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1407; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1408; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1409; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1410; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1411; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1412; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1413;
1414; GFX940-TGSPLIT-LABEL: flat_workgroup_release_atomicrmw:
1415; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1416; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1417; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1418; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1419; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1420; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1421; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1422; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1423; GFX940-TGSPLIT-NEXT:    s_endpgm
1424;
1425; GFX11-WGP-LABEL: flat_workgroup_release_atomicrmw:
1426; GFX11-WGP:       ; %bb.0: ; %entry
1427; GFX11-WGP-NEXT:    s_clause 0x1
1428; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1429; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1430; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1431; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1432; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1433; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1434; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1435; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1436; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1437; GFX11-WGP-NEXT:    s_endpgm
1438;
1439; GFX11-CU-LABEL: flat_workgroup_release_atomicrmw:
1440; GFX11-CU:       ; %bb.0: ; %entry
1441; GFX11-CU-NEXT:    s_clause 0x1
1442; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1443; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1444; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1445; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1446; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1447; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1448; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1449; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1450; GFX11-CU-NEXT:    s_endpgm
1451    i32* %out, i32 %in) {
1452entry:
1453  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") release
1454  ret void
1455}
1456
1457define amdgpu_kernel void @flat_workgroup_acq_rel_atomicrmw(
1458; GFX7-LABEL: flat_workgroup_acq_rel_atomicrmw:
1459; GFX7:       ; %bb.0: ; %entry
1460; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1461; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1462; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1463; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1464; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1465; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1466; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1467; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1468; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1469; GFX7-NEXT:    s_endpgm
1470;
1471; GFX10-WGP-LABEL: flat_workgroup_acq_rel_atomicrmw:
1472; GFX10-WGP:       ; %bb.0: ; %entry
1473; GFX10-WGP-NEXT:    s_clause 0x1
1474; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1475; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1476; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1477; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1478; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1479; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1480; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1481; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1482; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1483; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1484; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1485; GFX10-WGP-NEXT:    buffer_gl0_inv
1486; GFX10-WGP-NEXT:    s_endpgm
1487;
1488; GFX10-CU-LABEL: flat_workgroup_acq_rel_atomicrmw:
1489; GFX10-CU:       ; %bb.0: ; %entry
1490; GFX10-CU-NEXT:    s_clause 0x1
1491; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1492; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1493; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1494; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1495; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1496; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1497; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1498; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1499; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1500; GFX10-CU-NEXT:    s_endpgm
1501;
1502; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_atomicrmw:
1503; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1504; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1505; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1506; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1507; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1508; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1509; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1510; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1511; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1512; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1513; SKIP-CACHE-INV-NEXT:    s_endpgm
1514;
1515; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw:
1516; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1517; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1518; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1519; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1520; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1521; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1522; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1523; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1524; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1525; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1526;
1527; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw:
1528; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1529; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1530; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1531; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1532; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1533; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1534; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1535; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1536; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1537; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1538; GFX90A-TGSPLIT-NEXT:    s_endpgm
1539;
1540; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw:
1541; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1542; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1543; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1544; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1545; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1546; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1547; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1548; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1549; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1550; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1551;
1552; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw:
1553; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1554; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1555; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1556; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1557; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1558; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1559; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1560; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1561; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1562; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
1563; GFX940-TGSPLIT-NEXT:    s_endpgm
1564;
1565; GFX11-WGP-LABEL: flat_workgroup_acq_rel_atomicrmw:
1566; GFX11-WGP:       ; %bb.0: ; %entry
1567; GFX11-WGP-NEXT:    s_clause 0x1
1568; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1569; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1570; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1571; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1572; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1573; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1574; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1575; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1576; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1577; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1578; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1579; GFX11-WGP-NEXT:    buffer_gl0_inv
1580; GFX11-WGP-NEXT:    s_endpgm
1581;
1582; GFX11-CU-LABEL: flat_workgroup_acq_rel_atomicrmw:
1583; GFX11-CU:       ; %bb.0: ; %entry
1584; GFX11-CU-NEXT:    s_clause 0x1
1585; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1586; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1587; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1588; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1589; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1590; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1591; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1592; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1593; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1594; GFX11-CU-NEXT:    s_endpgm
1595    i32* %out, i32 %in) {
1596entry:
1597  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acq_rel
1598  ret void
1599}
1600
1601define amdgpu_kernel void @flat_workgroup_seq_cst_atomicrmw(
1602; GFX7-LABEL: flat_workgroup_seq_cst_atomicrmw:
1603; GFX7:       ; %bb.0: ; %entry
1604; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1605; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1606; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1607; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1608; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1609; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1610; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1611; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1612; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1613; GFX7-NEXT:    s_endpgm
1614;
1615; GFX10-WGP-LABEL: flat_workgroup_seq_cst_atomicrmw:
1616; GFX10-WGP:       ; %bb.0: ; %entry
1617; GFX10-WGP-NEXT:    s_clause 0x1
1618; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1619; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1620; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1621; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1622; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1623; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1624; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1625; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1626; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1627; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1628; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1629; GFX10-WGP-NEXT:    buffer_gl0_inv
1630; GFX10-WGP-NEXT:    s_endpgm
1631;
1632; GFX10-CU-LABEL: flat_workgroup_seq_cst_atomicrmw:
1633; GFX10-CU:       ; %bb.0: ; %entry
1634; GFX10-CU-NEXT:    s_clause 0x1
1635; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1636; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1637; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1638; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1639; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1640; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1641; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1642; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1643; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1644; GFX10-CU-NEXT:    s_endpgm
1645;
1646; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_atomicrmw:
1647; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1648; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1649; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1650; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1651; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1652; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1653; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1654; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1655; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1656; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1657; SKIP-CACHE-INV-NEXT:    s_endpgm
1658;
1659; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw:
1660; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1661; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1662; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1663; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1664; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1665; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1666; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1667; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1668; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1669; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1670;
1671; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw:
1672; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1673; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1674; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1675; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1676; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1677; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1678; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1679; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1680; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1681; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1682; GFX90A-TGSPLIT-NEXT:    s_endpgm
1683;
1684; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw:
1685; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1686; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1687; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1688; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1689; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1690; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1691; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1692; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1693; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1694; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1695;
1696; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw:
1697; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1698; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1699; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1700; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1701; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1702; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1703; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1704; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1705; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1706; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
1707; GFX940-TGSPLIT-NEXT:    s_endpgm
1708;
1709; GFX11-WGP-LABEL: flat_workgroup_seq_cst_atomicrmw:
1710; GFX11-WGP:       ; %bb.0: ; %entry
1711; GFX11-WGP-NEXT:    s_clause 0x1
1712; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1713; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1714; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1715; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1716; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1717; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1718; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1719; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1720; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1721; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1722; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1723; GFX11-WGP-NEXT:    buffer_gl0_inv
1724; GFX11-WGP-NEXT:    s_endpgm
1725;
1726; GFX11-CU-LABEL: flat_workgroup_seq_cst_atomicrmw:
1727; GFX11-CU:       ; %bb.0: ; %entry
1728; GFX11-CU-NEXT:    s_clause 0x1
1729; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1730; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1731; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1732; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1733; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1734; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1735; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1736; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1737; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1738; GFX11-CU-NEXT:    s_endpgm
1739    i32* %out, i32 %in) {
1740entry:
1741  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") seq_cst
1742  ret void
1743}
1744
1745define amdgpu_kernel void @flat_workgroup_acquire_ret_atomicrmw(
1746; GFX7-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1747; GFX7:       ; %bb.0: ; %entry
1748; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1749; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1750; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1751; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1752; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1753; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1754; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1755; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1756; GFX7-NEXT:    flat_store_dword v[0:1], v2
1757; GFX7-NEXT:    s_endpgm
1758;
1759; GFX10-WGP-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1760; GFX10-WGP:       ; %bb.0: ; %entry
1761; GFX10-WGP-NEXT:    s_clause 0x1
1762; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1763; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1764; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1765; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1766; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1767; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1768; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1769; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1770; GFX10-WGP-NEXT:    buffer_gl0_inv
1771; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1772; GFX10-WGP-NEXT:    s_endpgm
1773;
1774; GFX10-CU-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1775; GFX10-CU:       ; %bb.0: ; %entry
1776; GFX10-CU-NEXT:    s_clause 0x1
1777; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1778; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1779; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1780; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1781; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1782; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1783; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1784; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1785; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1786; GFX10-CU-NEXT:    s_endpgm
1787;
1788; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1789; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1790; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1791; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1792; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1793; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1794; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1795; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1796; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1797; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1798; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1799; SKIP-CACHE-INV-NEXT:    s_endpgm
1800;
1801; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1802; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1803; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1804; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1805; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1806; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1807; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1808; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1809; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1810; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1811; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1812;
1813; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1814; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1815; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1816; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1817; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1818; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1819; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1820; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1821; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1822; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1823; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1824; GFX90A-TGSPLIT-NEXT:    s_endpgm
1825;
1826; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1827; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1828; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1829; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1830; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1831; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1832; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1833; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1834; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1835; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1836; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1837;
1838; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1839; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1840; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1841; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1842; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1843; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1844; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1845; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1846; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1847; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
1848; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1849; GFX940-TGSPLIT-NEXT:    s_endpgm
1850;
1851; GFX11-WGP-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1852; GFX11-WGP:       ; %bb.0: ; %entry
1853; GFX11-WGP-NEXT:    s_clause 0x1
1854; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1855; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1856; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1857; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1858; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1859; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1860; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1861; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1862; GFX11-WGP-NEXT:    buffer_gl0_inv
1863; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
1864; GFX11-WGP-NEXT:    s_endpgm
1865;
1866; GFX11-CU-LABEL: flat_workgroup_acquire_ret_atomicrmw:
1867; GFX11-CU:       ; %bb.0: ; %entry
1868; GFX11-CU-NEXT:    s_clause 0x1
1869; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1870; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1871; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1872; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1873; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1874; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1875; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1876; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1877; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1878; GFX11-CU-NEXT:    s_endpgm
1879    i32* %out, i32 %in) {
1880entry:
1881  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acquire
1882  store i32 %val, i32* %out, align 4
1883  ret void
1884}
1885
1886define amdgpu_kernel void @flat_workgroup_acq_rel_ret_atomicrmw(
1887; GFX7-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1888; GFX7:       ; %bb.0: ; %entry
1889; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1890; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1891; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1892; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1893; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1894; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1895; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1896; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1897; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1898; GFX7-NEXT:    flat_store_dword v[0:1], v2
1899; GFX7-NEXT:    s_endpgm
1900;
1901; GFX10-WGP-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1902; GFX10-WGP:       ; %bb.0: ; %entry
1903; GFX10-WGP-NEXT:    s_clause 0x1
1904; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1905; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1906; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1907; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1908; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1909; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1910; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1911; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1912; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1913; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1914; GFX10-WGP-NEXT:    buffer_gl0_inv
1915; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1916; GFX10-WGP-NEXT:    s_endpgm
1917;
1918; GFX10-CU-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1919; GFX10-CU:       ; %bb.0: ; %entry
1920; GFX10-CU-NEXT:    s_clause 0x1
1921; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1922; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1923; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1924; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1925; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1926; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1927; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1928; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1929; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1930; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1931; GFX10-CU-NEXT:    s_endpgm
1932;
1933; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1934; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1935; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1936; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1937; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1938; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1939; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1940; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1941; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1942; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1943; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1944; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1945; SKIP-CACHE-INV-NEXT:    s_endpgm
1946;
1947; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1948; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1949; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1950; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1951; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1952; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1953; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1954; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1955; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1956; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1957; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1958; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1959;
1960; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1961; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1962; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1963; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1964; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1965; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1966; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1967; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1968; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1969; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1970; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1971; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1972; GFX90A-TGSPLIT-NEXT:    s_endpgm
1973;
1974; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1975; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1976; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1977; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1978; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1979; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1980; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1981; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1982; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1983; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1984; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1985; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1986;
1987; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
1988; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1989; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1990; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1991; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1992; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1993; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1994; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1995; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1996; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1997; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
1998; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1999; GFX940-TGSPLIT-NEXT:    s_endpgm
2000;
2001; GFX11-WGP-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
2002; GFX11-WGP:       ; %bb.0: ; %entry
2003; GFX11-WGP-NEXT:    s_clause 0x1
2004; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2005; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
2006; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2007; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2008; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2009; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2010; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2011; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2012; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2013; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2014; GFX11-WGP-NEXT:    buffer_gl0_inv
2015; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
2016; GFX11-WGP-NEXT:    s_endpgm
2017;
2018; GFX11-CU-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
2019; GFX11-CU:       ; %bb.0: ; %entry
2020; GFX11-CU-NEXT:    s_clause 0x1
2021; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2022; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
2023; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2024; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2025; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2026; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2027; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2028; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2029; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2030; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
2031; GFX11-CU-NEXT:    s_endpgm
2032    i32* %out, i32 %in) {
2033entry:
2034  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acq_rel
2035  store i32 %val, i32* %out, align 4
2036  ret void
2037}
2038
2039define amdgpu_kernel void @flat_workgroup_seq_cst_ret_atomicrmw(
2040; GFX7-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2041; GFX7:       ; %bb.0: ; %entry
2042; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2043; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
2044; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2045; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2046; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2047; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2048; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2049; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2050; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2051; GFX7-NEXT:    flat_store_dword v[0:1], v2
2052; GFX7-NEXT:    s_endpgm
2053;
2054; GFX10-WGP-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2055; GFX10-WGP:       ; %bb.0: ; %entry
2056; GFX10-WGP-NEXT:    s_clause 0x1
2057; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2058; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
2059; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2060; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2061; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2062; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2063; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2064; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2065; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2066; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2067; GFX10-WGP-NEXT:    buffer_gl0_inv
2068; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2069; GFX10-WGP-NEXT:    s_endpgm
2070;
2071; GFX10-CU-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2072; GFX10-CU:       ; %bb.0: ; %entry
2073; GFX10-CU-NEXT:    s_clause 0x1
2074; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2075; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
2076; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2077; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2078; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2079; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2080; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2081; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2082; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2083; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2084; GFX10-CU-NEXT:    s_endpgm
2085;
2086; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2087; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2088; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2089; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
2090; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2091; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2092; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2093; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2094; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2095; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2096; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2097; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2098; SKIP-CACHE-INV-NEXT:    s_endpgm
2099;
2100; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2101; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2102; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2103; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2104; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2105; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2106; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
2107; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2108; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2109; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2110; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2111; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2112;
2113; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2114; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2115; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2116; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2117; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2118; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2119; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
2120; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2121; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2122; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2123; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2124; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2125; GFX90A-TGSPLIT-NEXT:    s_endpgm
2126;
2127; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2128; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2129; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2130; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2131; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2132; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2133; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2134; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2135; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
2136; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2137; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2138; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2139;
2140; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2141; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2142; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2143; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2144; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2145; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2146; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2147; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2148; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
2149; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2150; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
2151; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2152; GFX940-TGSPLIT-NEXT:    s_endpgm
2153;
2154; GFX11-WGP-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2155; GFX11-WGP:       ; %bb.0: ; %entry
2156; GFX11-WGP-NEXT:    s_clause 0x1
2157; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2158; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
2159; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2160; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2161; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2162; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2163; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2164; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2165; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2166; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2167; GFX11-WGP-NEXT:    buffer_gl0_inv
2168; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
2169; GFX11-WGP-NEXT:    s_endpgm
2170;
2171; GFX11-CU-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2172; GFX11-CU:       ; %bb.0: ; %entry
2173; GFX11-CU-NEXT:    s_clause 0x1
2174; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2175; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
2176; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2177; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2178; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2179; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2180; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2181; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2182; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2183; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
2184; GFX11-CU-NEXT:    s_endpgm
2185    i32* %out, i32 %in) {
2186entry:
2187  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") seq_cst
2188  store i32 %val, i32* %out, align 4
2189  ret void
2190}
2191
2192define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_cmpxchg(
2193; GFX7-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2194; GFX7:       ; %bb.0: ; %entry
2195; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2196; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2197; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2198; GFX7-NEXT:    s_add_u32 s0, s0, 16
2199; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2200; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2201; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2202; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2203; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2204; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2205; GFX7-NEXT:    s_endpgm
2206;
2207; GFX10-WGP-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2208; GFX10-WGP:       ; %bb.0: ; %entry
2209; GFX10-WGP-NEXT:    s_clause 0x1
2210; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2211; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2212; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2213; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2214; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2215; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2216; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2217; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2218; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2219; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2220; GFX10-WGP-NEXT:    s_endpgm
2221;
2222; GFX10-CU-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2223; GFX10-CU:       ; %bb.0: ; %entry
2224; GFX10-CU-NEXT:    s_clause 0x1
2225; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2226; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2227; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2228; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2229; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2230; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2231; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2232; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2233; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2234; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2235; GFX10-CU-NEXT:    s_endpgm
2236;
2237; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2238; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2239; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2240; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2241; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2242; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2243; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2244; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2245; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2246; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2247; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2248; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2249; SKIP-CACHE-INV-NEXT:    s_endpgm
2250;
2251; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2252; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2253; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2254; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2255; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2256; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2257; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2258; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2259; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2260;
2261; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2262; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2263; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2264; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2265; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2266; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2267; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2268; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2269; GFX90A-TGSPLIT-NEXT:    s_endpgm
2270;
2271; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2272; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2273; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2274; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2275; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2276; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2277; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2278; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2279; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2280;
2281; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2282; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2283; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2284; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2285; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2286; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2287; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2288; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2289; GFX940-TGSPLIT-NEXT:    s_endpgm
2290;
2291; GFX11-WGP-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2292; GFX11-WGP:       ; %bb.0: ; %entry
2293; GFX11-WGP-NEXT:    s_clause 0x1
2294; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2295; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2296; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2297; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2298; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2299; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2300; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2301; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2302; GFX11-WGP-NEXT:    s_endpgm
2303;
2304; GFX11-CU-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2305; GFX11-CU:       ; %bb.0: ; %entry
2306; GFX11-CU-NEXT:    s_clause 0x1
2307; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2308; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2309; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2310; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2311; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2312; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2313; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2314; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2315; GFX11-CU-NEXT:    s_endpgm
2316    i32* %out, i32 %in, i32 %old) {
2317entry:
2318  %gep = getelementptr i32, i32* %out, i32 4
2319  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic
2320  ret void
2321}
2322
2323define amdgpu_kernel void @flat_workgroup_acquire_monotonic_cmpxchg(
2324; GFX7-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
2325; GFX7:       ; %bb.0: ; %entry
2326; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2327; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2328; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2329; GFX7-NEXT:    s_add_u32 s0, s0, 16
2330; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2331; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2332; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2333; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2334; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2335; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2336; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2337; GFX7-NEXT:    s_endpgm
2338;
2339; GFX10-WGP-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
2340; GFX10-WGP:       ; %bb.0: ; %entry
2341; GFX10-WGP-NEXT:    s_clause 0x1
2342; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2343; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2344; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2345; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2346; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2347; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2348; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2349; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2350; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2351; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2352; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2353; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2354; GFX10-WGP-NEXT:    buffer_gl0_inv
2355; GFX10-WGP-NEXT:    s_endpgm
2356;
2357; GFX10-CU-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
2358; GFX10-CU:       ; %bb.0: ; %entry
2359; GFX10-CU-NEXT:    s_clause 0x1
2360; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2361; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2362; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2363; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2364; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2365; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2366; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2367; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2368; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2369; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2370; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2371; GFX10-CU-NEXT:    s_endpgm
2372;
2373; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
2374; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2375; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2376; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2377; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2378; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2379; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2380; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2381; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2382; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2383; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2384; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2385; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2386; SKIP-CACHE-INV-NEXT:    s_endpgm
2387;
2388; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
2389; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2390; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2391; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2392; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2393; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2394; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2395; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2396; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2397; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2398;
2399; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
2400; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2401; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2402; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2403; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2404; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2405; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2406; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2407; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2408; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2409; GFX90A-TGSPLIT-NEXT:    s_endpgm
2410;
2411; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
2412; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2413; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2414; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2415; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2416; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2417; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2418; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2419; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2420; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2421;
2422; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
2423; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2424; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2425; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2426; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2427; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2428; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2429; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2430; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2431; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
2432; GFX940-TGSPLIT-NEXT:    s_endpgm
2433;
2434; GFX11-WGP-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
2435; GFX11-WGP:       ; %bb.0: ; %entry
2436; GFX11-WGP-NEXT:    s_clause 0x1
2437; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2438; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2439; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2440; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2441; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2442; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2443; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2444; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2445; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2446; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2447; GFX11-WGP-NEXT:    buffer_gl0_inv
2448; GFX11-WGP-NEXT:    s_endpgm
2449;
2450; GFX11-CU-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
2451; GFX11-CU:       ; %bb.0: ; %entry
2452; GFX11-CU-NEXT:    s_clause 0x1
2453; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2454; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2455; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2456; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2457; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2458; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2459; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2460; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2461; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2462; GFX11-CU-NEXT:    s_endpgm
2463    i32* %out, i32 %in, i32 %old) {
2464entry:
2465  %gep = getelementptr i32, i32* %out, i32 4
2466  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
2467  ret void
2468}
2469
2470define amdgpu_kernel void @flat_workgroup_release_monotonic_cmpxchg(
2471; GFX7-LABEL: flat_workgroup_release_monotonic_cmpxchg:
2472; GFX7:       ; %bb.0: ; %entry
2473; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2474; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2475; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2476; GFX7-NEXT:    s_add_u32 s0, s0, 16
2477; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2478; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2479; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2480; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2481; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2482; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2483; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2484; GFX7-NEXT:    s_endpgm
2485;
2486; GFX10-WGP-LABEL: flat_workgroup_release_monotonic_cmpxchg:
2487; GFX10-WGP:       ; %bb.0: ; %entry
2488; GFX10-WGP-NEXT:    s_clause 0x1
2489; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2490; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2491; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2492; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2493; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2494; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2495; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2496; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2497; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2498; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2499; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2500; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2501; GFX10-WGP-NEXT:    s_endpgm
2502;
2503; GFX10-CU-LABEL: flat_workgroup_release_monotonic_cmpxchg:
2504; GFX10-CU:       ; %bb.0: ; %entry
2505; GFX10-CU-NEXT:    s_clause 0x1
2506; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2507; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2508; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2509; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2510; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2511; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2512; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2513; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2514; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2515; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2516; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2517; GFX10-CU-NEXT:    s_endpgm
2518;
2519; SKIP-CACHE-INV-LABEL: flat_workgroup_release_monotonic_cmpxchg:
2520; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2521; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2522; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2523; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2524; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2525; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2526; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2527; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2528; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2529; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2530; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2531; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2532; SKIP-CACHE-INV-NEXT:    s_endpgm
2533;
2534; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg:
2535; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2536; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2537; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2538; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2539; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2540; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2541; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2542; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2543; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2544;
2545; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg:
2546; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2547; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2548; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2549; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2550; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2551; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2552; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2553; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2554; GFX90A-TGSPLIT-NEXT:    s_endpgm
2555;
2556; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg:
2557; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2558; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2559; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2560; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2561; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2562; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2563; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2564; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2565; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2566;
2567; GFX940-TGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg:
2568; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2569; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2570; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2571; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2572; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2573; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2574; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2575; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2576; GFX940-TGSPLIT-NEXT:    s_endpgm
2577;
2578; GFX11-WGP-LABEL: flat_workgroup_release_monotonic_cmpxchg:
2579; GFX11-WGP:       ; %bb.0: ; %entry
2580; GFX11-WGP-NEXT:    s_clause 0x1
2581; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2582; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2583; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2584; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2585; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2586; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2587; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2588; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2589; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2590; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2591; GFX11-WGP-NEXT:    s_endpgm
2592;
2593; GFX11-CU-LABEL: flat_workgroup_release_monotonic_cmpxchg:
2594; GFX11-CU:       ; %bb.0: ; %entry
2595; GFX11-CU-NEXT:    s_clause 0x1
2596; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2597; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2598; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2599; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2600; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2601; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2602; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2603; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2604; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2605; GFX11-CU-NEXT:    s_endpgm
2606    i32* %out, i32 %in, i32 %old) {
2607entry:
2608  %gep = getelementptr i32, i32* %out, i32 4
2609  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic
2610  ret void
2611}
2612
2613define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_cmpxchg(
2614; GFX7-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
2615; GFX7:       ; %bb.0: ; %entry
2616; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2617; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2618; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2619; GFX7-NEXT:    s_add_u32 s0, s0, 16
2620; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2621; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2622; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2623; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2624; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2625; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2626; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2627; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2628; GFX7-NEXT:    s_endpgm
2629;
2630; GFX10-WGP-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
2631; GFX10-WGP:       ; %bb.0: ; %entry
2632; GFX10-WGP-NEXT:    s_clause 0x1
2633; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2634; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2635; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2636; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2637; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2638; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2639; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2640; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2641; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2642; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2643; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2644; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2645; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2646; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2647; GFX10-WGP-NEXT:    buffer_gl0_inv
2648; GFX10-WGP-NEXT:    s_endpgm
2649;
2650; GFX10-CU-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
2651; GFX10-CU:       ; %bb.0: ; %entry
2652; GFX10-CU-NEXT:    s_clause 0x1
2653; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2654; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2655; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2656; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2657; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2658; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2659; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2660; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2661; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2662; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2663; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2664; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2665; GFX10-CU-NEXT:    s_endpgm
2666;
2667; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
2668; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2669; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2670; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2671; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2672; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2673; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2674; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2675; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2676; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2677; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2678; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2679; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2680; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2681; SKIP-CACHE-INV-NEXT:    s_endpgm
2682;
2683; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
2684; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2685; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2686; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2687; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2688; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2689; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2690; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2691; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2692; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2693; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2694;
2695; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
2696; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2697; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2698; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2699; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2700; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2701; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2702; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2703; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2704; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2705; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2706; GFX90A-TGSPLIT-NEXT:    s_endpgm
2707;
2708; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
2709; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2710; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2711; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2712; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2713; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2714; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2715; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2716; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2717; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2718; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2719;
2720; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
2721; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2722; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2723; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2724; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2725; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2726; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2727; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2728; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2729; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2730; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
2731; GFX940-TGSPLIT-NEXT:    s_endpgm
2732;
2733; GFX11-WGP-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
2734; GFX11-WGP:       ; %bb.0: ; %entry
2735; GFX11-WGP-NEXT:    s_clause 0x1
2736; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2737; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2738; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2739; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2740; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2741; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2742; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2743; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2744; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2745; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2746; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2747; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2748; GFX11-WGP-NEXT:    buffer_gl0_inv
2749; GFX11-WGP-NEXT:    s_endpgm
2750;
2751; GFX11-CU-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
2752; GFX11-CU:       ; %bb.0: ; %entry
2753; GFX11-CU-NEXT:    s_clause 0x1
2754; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2755; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2756; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2757; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2758; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2759; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2760; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2761; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2762; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2763; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2764; GFX11-CU-NEXT:    s_endpgm
2765    i32* %out, i32 %in, i32 %old) {
2766entry:
2767  %gep = getelementptr i32, i32* %out, i32 4
2768  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
2769  ret void
2770}
2771
2772define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_cmpxchg(
2773; GFX7-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
2774; GFX7:       ; %bb.0: ; %entry
2775; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2776; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2777; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2778; GFX7-NEXT:    s_add_u32 s0, s0, 16
2779; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2780; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2781; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2782; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2783; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2784; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2785; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2786; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2787; GFX7-NEXT:    s_endpgm
2788;
2789; GFX10-WGP-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
2790; GFX10-WGP:       ; %bb.0: ; %entry
2791; GFX10-WGP-NEXT:    s_clause 0x1
2792; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2793; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2794; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2795; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2796; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2797; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2798; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2799; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2800; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2801; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2802; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2803; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2804; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2805; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2806; GFX10-WGP-NEXT:    buffer_gl0_inv
2807; GFX10-WGP-NEXT:    s_endpgm
2808;
2809; GFX10-CU-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
2810; GFX10-CU:       ; %bb.0: ; %entry
2811; GFX10-CU-NEXT:    s_clause 0x1
2812; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2813; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2814; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2815; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2816; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2817; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2818; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2819; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2820; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2821; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2822; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2823; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2824; GFX10-CU-NEXT:    s_endpgm
2825;
2826; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
2827; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2828; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2829; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2830; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2831; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2832; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2833; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2834; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2835; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2836; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2837; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2838; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2839; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2840; SKIP-CACHE-INV-NEXT:    s_endpgm
2841;
2842; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
2843; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2844; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2845; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2846; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2847; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2848; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2849; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2850; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2851; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2852; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2853;
2854; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
2855; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2856; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2857; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2858; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2859; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2860; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2861; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2862; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2863; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2864; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2865; GFX90A-TGSPLIT-NEXT:    s_endpgm
2866;
2867; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
2868; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2869; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2870; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2871; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2872; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2873; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2874; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2875; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2876; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2877; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2878;
2879; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
2880; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2881; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2882; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2883; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2884; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2885; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2886; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2887; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2888; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2889; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
2890; GFX940-TGSPLIT-NEXT:    s_endpgm
2891;
2892; GFX11-WGP-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
2893; GFX11-WGP:       ; %bb.0: ; %entry
2894; GFX11-WGP-NEXT:    s_clause 0x1
2895; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2896; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2897; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2898; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2899; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2900; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2901; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2902; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2903; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2904; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2905; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2906; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2907; GFX11-WGP-NEXT:    buffer_gl0_inv
2908; GFX11-WGP-NEXT:    s_endpgm
2909;
2910; GFX11-CU-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
2911; GFX11-CU:       ; %bb.0: ; %entry
2912; GFX11-CU-NEXT:    s_clause 0x1
2913; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2914; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2915; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2916; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2917; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2918; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2919; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2920; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2921; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2922; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2923; GFX11-CU-NEXT:    s_endpgm
2924    i32* %out, i32 %in, i32 %old) {
2925entry:
2926  %gep = getelementptr i32, i32* %out, i32 4
2927  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
2928  ret void
2929}
2930
2931define amdgpu_kernel void @flat_workgroup_monotonic_acquire_cmpxchg(
2932; GFX7-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
2933; GFX7:       ; %bb.0: ; %entry
2934; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2935; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2936; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2937; GFX7-NEXT:    s_add_u32 s0, s0, 16
2938; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2939; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2940; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2941; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2942; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2943; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2944; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2945; GFX7-NEXT:    s_endpgm
2946;
2947; GFX10-WGP-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
2948; GFX10-WGP:       ; %bb.0: ; %entry
2949; GFX10-WGP-NEXT:    s_clause 0x1
2950; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2951; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2952; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2953; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2954; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2955; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2956; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2957; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2958; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2959; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2960; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2961; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2962; GFX10-WGP-NEXT:    buffer_gl0_inv
2963; GFX10-WGP-NEXT:    s_endpgm
2964;
2965; GFX10-CU-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
2966; GFX10-CU:       ; %bb.0: ; %entry
2967; GFX10-CU-NEXT:    s_clause 0x1
2968; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2969; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2970; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2971; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2972; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2973; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2974; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2975; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2976; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2977; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2978; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2979; GFX10-CU-NEXT:    s_endpgm
2980;
2981; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
2982; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2983; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2984; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2985; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2986; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2987; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2988; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2989; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2990; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2991; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2992; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2993; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2994; SKIP-CACHE-INV-NEXT:    s_endpgm
2995;
2996; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
2997; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2998; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2999; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3000; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3001; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3002; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3003; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3004; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3005; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3006;
3007; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
3008; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3009; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3010; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3011; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3012; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3013; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3014; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3015; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3016; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3017; GFX90A-TGSPLIT-NEXT:    s_endpgm
3018;
3019; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
3020; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3021; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3022; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3023; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3024; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3025; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3026; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3027; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3028; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3029;
3030; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
3031; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3032; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3033; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3034; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3035; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3036; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3037; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3038; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3039; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
3040; GFX940-TGSPLIT-NEXT:    s_endpgm
3041;
3042; GFX11-WGP-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
3043; GFX11-WGP:       ; %bb.0: ; %entry
3044; GFX11-WGP-NEXT:    s_clause 0x1
3045; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3046; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3047; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3048; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3049; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3050; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3051; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3052; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3053; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3054; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3055; GFX11-WGP-NEXT:    buffer_gl0_inv
3056; GFX11-WGP-NEXT:    s_endpgm
3057;
3058; GFX11-CU-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
3059; GFX11-CU:       ; %bb.0: ; %entry
3060; GFX11-CU-NEXT:    s_clause 0x1
3061; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3062; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3063; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3064; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3065; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3066; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3067; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3068; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3069; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3070; GFX11-CU-NEXT:    s_endpgm
3071    i32* %out, i32 %in, i32 %old) {
3072entry:
3073  %gep = getelementptr i32, i32* %out, i32 4
3074  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic acquire
3075  ret void
3076}
3077
3078define amdgpu_kernel void @flat_workgroup_acquire_acquire_cmpxchg(
3079; GFX7-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
3080; GFX7:       ; %bb.0: ; %entry
3081; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3082; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3083; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3084; GFX7-NEXT:    s_add_u32 s0, s0, 16
3085; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3086; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3087; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3088; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3089; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3090; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3091; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3092; GFX7-NEXT:    s_endpgm
3093;
3094; GFX10-WGP-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
3095; GFX10-WGP:       ; %bb.0: ; %entry
3096; GFX10-WGP-NEXT:    s_clause 0x1
3097; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3098; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3099; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3100; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3101; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3102; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3103; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3104; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3105; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3106; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3107; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3108; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3109; GFX10-WGP-NEXT:    buffer_gl0_inv
3110; GFX10-WGP-NEXT:    s_endpgm
3111;
3112; GFX10-CU-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
3113; GFX10-CU:       ; %bb.0: ; %entry
3114; GFX10-CU-NEXT:    s_clause 0x1
3115; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3116; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3117; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3118; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3119; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3120; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3121; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3122; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3123; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3124; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3125; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3126; GFX10-CU-NEXT:    s_endpgm
3127;
3128; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
3129; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3130; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3131; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3132; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3133; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3134; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3135; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3136; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3137; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3138; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3139; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3140; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3141; SKIP-CACHE-INV-NEXT:    s_endpgm
3142;
3143; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
3144; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3145; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3146; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3147; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3148; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3149; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3150; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3151; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3152; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3153;
3154; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
3155; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3156; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3157; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3158; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3159; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3160; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3161; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3162; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3163; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3164; GFX90A-TGSPLIT-NEXT:    s_endpgm
3165;
3166; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
3167; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3168; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3169; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3170; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3171; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3172; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3173; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3174; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3175; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3176;
3177; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
3178; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3179; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3180; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3181; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3182; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3183; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3184; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3185; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3186; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
3187; GFX940-TGSPLIT-NEXT:    s_endpgm
3188;
3189; GFX11-WGP-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
3190; GFX11-WGP:       ; %bb.0: ; %entry
3191; GFX11-WGP-NEXT:    s_clause 0x1
3192; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3193; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3194; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3195; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3196; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3197; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3198; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3199; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3200; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3201; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3202; GFX11-WGP-NEXT:    buffer_gl0_inv
3203; GFX11-WGP-NEXT:    s_endpgm
3204;
3205; GFX11-CU-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
3206; GFX11-CU:       ; %bb.0: ; %entry
3207; GFX11-CU-NEXT:    s_clause 0x1
3208; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3209; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3210; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3211; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3212; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3213; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3214; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3215; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3216; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3217; GFX11-CU-NEXT:    s_endpgm
3218    i32* %out, i32 %in, i32 %old) {
3219entry:
3220  %gep = getelementptr i32, i32* %out, i32 4
3221  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
3222  ret void
3223}
3224
3225define amdgpu_kernel void @flat_workgroup_release_acquire_cmpxchg(
3226; GFX7-LABEL: flat_workgroup_release_acquire_cmpxchg:
3227; GFX7:       ; %bb.0: ; %entry
3228; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3229; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3230; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3231; GFX7-NEXT:    s_add_u32 s0, s0, 16
3232; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3233; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3234; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3235; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3236; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3237; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3238; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3239; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3240; GFX7-NEXT:    s_endpgm
3241;
3242; GFX10-WGP-LABEL: flat_workgroup_release_acquire_cmpxchg:
3243; GFX10-WGP:       ; %bb.0: ; %entry
3244; GFX10-WGP-NEXT:    s_clause 0x1
3245; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3246; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3247; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3248; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3249; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3250; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3251; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3252; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3253; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3254; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3255; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3256; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3257; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3258; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3259; GFX10-WGP-NEXT:    buffer_gl0_inv
3260; GFX10-WGP-NEXT:    s_endpgm
3261;
3262; GFX10-CU-LABEL: flat_workgroup_release_acquire_cmpxchg:
3263; GFX10-CU:       ; %bb.0: ; %entry
3264; GFX10-CU-NEXT:    s_clause 0x1
3265; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3266; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3267; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3268; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3269; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3270; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3271; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3272; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3273; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3274; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3275; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3276; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3277; GFX10-CU-NEXT:    s_endpgm
3278;
3279; SKIP-CACHE-INV-LABEL: flat_workgroup_release_acquire_cmpxchg:
3280; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3281; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3282; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3283; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3284; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3285; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3286; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3287; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3288; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3289; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3290; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3291; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3292; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3293; SKIP-CACHE-INV-NEXT:    s_endpgm
3294;
3295; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg:
3296; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3297; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3298; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3299; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3300; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3301; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3302; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3303; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3304; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3305; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3306;
3307; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg:
3308; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3309; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3310; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3311; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3312; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3313; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3314; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3315; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3316; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3317; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3318; GFX90A-TGSPLIT-NEXT:    s_endpgm
3319;
3320; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg:
3321; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3322; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3323; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3324; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3325; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3326; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3327; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3328; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3329; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3330; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3331;
3332; GFX940-TGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg:
3333; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3334; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3335; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3336; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3337; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3338; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3339; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3340; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3341; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3342; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
3343; GFX940-TGSPLIT-NEXT:    s_endpgm
3344;
3345; GFX11-WGP-LABEL: flat_workgroup_release_acquire_cmpxchg:
3346; GFX11-WGP:       ; %bb.0: ; %entry
3347; GFX11-WGP-NEXT:    s_clause 0x1
3348; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3349; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3350; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3351; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3352; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3353; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3354; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3355; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3356; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3357; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3358; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3359; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3360; GFX11-WGP-NEXT:    buffer_gl0_inv
3361; GFX11-WGP-NEXT:    s_endpgm
3362;
3363; GFX11-CU-LABEL: flat_workgroup_release_acquire_cmpxchg:
3364; GFX11-CU:       ; %bb.0: ; %entry
3365; GFX11-CU-NEXT:    s_clause 0x1
3366; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3367; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3368; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3369; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3370; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3371; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3372; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3373; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3374; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3375; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3376; GFX11-CU-NEXT:    s_endpgm
3377    i32* %out, i32 %in, i32 %old) {
3378entry:
3379  %gep = getelementptr i32, i32* %out, i32 4
3380  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
3381  ret void
3382}
3383
3384define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_cmpxchg(
3385; GFX7-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
3386; GFX7:       ; %bb.0: ; %entry
3387; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3388; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3389; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3390; GFX7-NEXT:    s_add_u32 s0, s0, 16
3391; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3392; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3393; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3394; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3395; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3396; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3397; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3398; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3399; GFX7-NEXT:    s_endpgm
3400;
3401; GFX10-WGP-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
3402; GFX10-WGP:       ; %bb.0: ; %entry
3403; GFX10-WGP-NEXT:    s_clause 0x1
3404; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3405; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3406; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3407; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3408; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3409; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3410; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3411; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3412; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3413; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3414; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3415; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3416; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3417; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3418; GFX10-WGP-NEXT:    buffer_gl0_inv
3419; GFX10-WGP-NEXT:    s_endpgm
3420;
3421; GFX10-CU-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
3422; GFX10-CU:       ; %bb.0: ; %entry
3423; GFX10-CU-NEXT:    s_clause 0x1
3424; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3425; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3426; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3427; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3428; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3429; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3430; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3431; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3432; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3433; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3434; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3435; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3436; GFX10-CU-NEXT:    s_endpgm
3437;
3438; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
3439; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3440; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3441; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3442; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3443; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3444; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3445; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3446; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3447; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3448; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3449; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3450; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3451; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3452; SKIP-CACHE-INV-NEXT:    s_endpgm
3453;
3454; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
3455; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3456; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3457; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3458; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3459; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3460; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3461; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3462; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3463; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3464; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3465;
3466; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
3467; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3468; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3469; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3470; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3471; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3472; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3473; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3474; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3475; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3476; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3477; GFX90A-TGSPLIT-NEXT:    s_endpgm
3478;
3479; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
3480; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3481; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3482; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3483; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3484; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3485; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3486; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3487; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3488; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3489; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3490;
3491; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
3492; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3493; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3494; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3495; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3496; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3497; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3498; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3499; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3500; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3501; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
3502; GFX940-TGSPLIT-NEXT:    s_endpgm
3503;
3504; GFX11-WGP-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
3505; GFX11-WGP:       ; %bb.0: ; %entry
3506; GFX11-WGP-NEXT:    s_clause 0x1
3507; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3508; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3509; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3510; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3511; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3512; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3513; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3514; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3515; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3516; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3517; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3518; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3519; GFX11-WGP-NEXT:    buffer_gl0_inv
3520; GFX11-WGP-NEXT:    s_endpgm
3521;
3522; GFX11-CU-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
3523; GFX11-CU:       ; %bb.0: ; %entry
3524; GFX11-CU-NEXT:    s_clause 0x1
3525; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3526; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3527; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3528; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3529; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3530; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3531; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3532; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3533; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3534; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3535; GFX11-CU-NEXT:    s_endpgm
3536    i32* %out, i32 %in, i32 %old) {
3537entry:
3538  %gep = getelementptr i32, i32* %out, i32 4
3539  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
3540  ret void
3541}
3542
3543define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_cmpxchg(
3544; GFX7-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
3545; GFX7:       ; %bb.0: ; %entry
3546; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3547; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3548; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3549; GFX7-NEXT:    s_add_u32 s0, s0, 16
3550; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3551; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3552; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3553; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3554; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3555; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3556; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3557; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3558; GFX7-NEXT:    s_endpgm
3559;
3560; GFX10-WGP-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
3561; GFX10-WGP:       ; %bb.0: ; %entry
3562; GFX10-WGP-NEXT:    s_clause 0x1
3563; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3564; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3565; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3566; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3567; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3568; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3569; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3570; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3571; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3572; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3573; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3574; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3575; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3576; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3577; GFX10-WGP-NEXT:    buffer_gl0_inv
3578; GFX10-WGP-NEXT:    s_endpgm
3579;
3580; GFX10-CU-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
3581; GFX10-CU:       ; %bb.0: ; %entry
3582; GFX10-CU-NEXT:    s_clause 0x1
3583; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3584; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3585; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3586; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3587; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3588; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3589; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3590; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3591; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3592; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3593; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3594; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3595; GFX10-CU-NEXT:    s_endpgm
3596;
3597; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
3598; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3599; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3600; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3601; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3602; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3603; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3604; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3605; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3606; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3607; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3608; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3609; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3610; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3611; SKIP-CACHE-INV-NEXT:    s_endpgm
3612;
3613; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
3614; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3615; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3616; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3617; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3618; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3619; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3620; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3621; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3622; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3623; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3624;
3625; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
3626; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3627; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3628; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3629; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3630; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3631; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3632; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3633; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3634; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3635; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3636; GFX90A-TGSPLIT-NEXT:    s_endpgm
3637;
3638; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
3639; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3640; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3641; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3642; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3643; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3644; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3645; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3646; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3647; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3648; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3649;
3650; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
3651; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3652; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3653; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3654; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3655; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3656; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3657; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3658; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3659; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3660; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
3661; GFX940-TGSPLIT-NEXT:    s_endpgm
3662;
3663; GFX11-WGP-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
3664; GFX11-WGP:       ; %bb.0: ; %entry
3665; GFX11-WGP-NEXT:    s_clause 0x1
3666; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3667; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3668; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3669; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3670; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3671; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3672; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3673; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3674; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3675; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3676; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3677; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3678; GFX11-WGP-NEXT:    buffer_gl0_inv
3679; GFX11-WGP-NEXT:    s_endpgm
3680;
3681; GFX11-CU-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
3682; GFX11-CU:       ; %bb.0: ; %entry
3683; GFX11-CU-NEXT:    s_clause 0x1
3684; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3685; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3686; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3687; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3688; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3689; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3690; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3691; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3692; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3693; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3694; GFX11-CU-NEXT:    s_endpgm
3695    i32* %out, i32 %in, i32 %old) {
3696entry:
3697  %gep = getelementptr i32, i32* %out, i32 4
3698  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
3699  ret void
3700}
3701
3702define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_cmpxchg(
3703; GFX7-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
3704; GFX7:       ; %bb.0: ; %entry
3705; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3706; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3707; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3708; GFX7-NEXT:    s_add_u32 s0, s0, 16
3709; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3710; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3711; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3712; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3713; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3714; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3715; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3716; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3717; GFX7-NEXT:    s_endpgm
3718;
3719; GFX10-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
3720; GFX10-WGP:       ; %bb.0: ; %entry
3721; GFX10-WGP-NEXT:    s_clause 0x1
3722; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3723; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3724; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3725; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3726; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3727; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3728; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3729; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3730; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3731; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3732; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3733; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3734; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3735; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3736; GFX10-WGP-NEXT:    buffer_gl0_inv
3737; GFX10-WGP-NEXT:    s_endpgm
3738;
3739; GFX10-CU-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
3740; GFX10-CU:       ; %bb.0: ; %entry
3741; GFX10-CU-NEXT:    s_clause 0x1
3742; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3743; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3744; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3745; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3746; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3747; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3748; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3749; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3750; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3751; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3752; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3753; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3754; GFX10-CU-NEXT:    s_endpgm
3755;
3756; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
3757; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3758; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3759; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3760; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3761; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3762; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3763; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3764; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3765; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3766; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3767; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3768; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3769; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3770; SKIP-CACHE-INV-NEXT:    s_endpgm
3771;
3772; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
3773; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3774; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3775; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3776; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3777; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3778; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3779; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3780; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3781; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3782; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3783;
3784; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
3785; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3786; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3787; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3788; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3789; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3790; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3791; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3792; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3793; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3794; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3795; GFX90A-TGSPLIT-NEXT:    s_endpgm
3796;
3797; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
3798; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3799; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3800; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3801; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3802; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3803; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3804; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3805; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3806; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3807; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3808;
3809; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
3810; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3811; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3812; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3813; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3814; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3815; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3816; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3817; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3818; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3819; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
3820; GFX940-TGSPLIT-NEXT:    s_endpgm
3821;
3822; GFX11-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
3823; GFX11-WGP:       ; %bb.0: ; %entry
3824; GFX11-WGP-NEXT:    s_clause 0x1
3825; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3826; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3827; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3828; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3829; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3830; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3831; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3832; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3833; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3834; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3835; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3836; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3837; GFX11-WGP-NEXT:    buffer_gl0_inv
3838; GFX11-WGP-NEXT:    s_endpgm
3839;
3840; GFX11-CU-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
3841; GFX11-CU:       ; %bb.0: ; %entry
3842; GFX11-CU-NEXT:    s_clause 0x1
3843; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3844; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3845; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3846; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3847; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3848; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3849; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3850; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3851; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3852; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3853; GFX11-CU-NEXT:    s_endpgm
3854    i32* %out, i32 %in, i32 %old) {
3855entry:
3856  %gep = getelementptr i32, i32* %out, i32 4
3857  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
3858  ret void
3859}
3860
3861define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_ret_cmpxchg(
3862; GFX7-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
3863; GFX7:       ; %bb.0: ; %entry
3864; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3865; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3866; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3867; GFX7-NEXT:    s_add_u32 s4, s0, 16
3868; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3869; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3870; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3871; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3872; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3873; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3874; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3875; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3876; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3877; GFX7-NEXT:    flat_store_dword v[0:1], v2
3878; GFX7-NEXT:    s_endpgm
3879;
3880; GFX10-WGP-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
3881; GFX10-WGP:       ; %bb.0: ; %entry
3882; GFX10-WGP-NEXT:    s_clause 0x1
3883; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3884; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3885; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3886; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3887; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3888; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3889; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3890; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3891; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3892; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3893; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3894; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3895; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3896; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3897; GFX10-WGP-NEXT:    s_endpgm
3898;
3899; GFX10-CU-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
3900; GFX10-CU:       ; %bb.0: ; %entry
3901; GFX10-CU-NEXT:    s_clause 0x1
3902; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3903; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3904; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3905; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3906; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3907; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3908; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3909; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3910; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3911; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3912; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3913; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3914; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3915; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3916; GFX10-CU-NEXT:    s_endpgm
3917;
3918; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
3919; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3920; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3921; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3922; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3923; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3924; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3925; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3926; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3927; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3928; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3929; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3930; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3931; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3932; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3933; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3934; SKIP-CACHE-INV-NEXT:    s_endpgm
3935;
3936; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
3937; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3938; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3939; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3940; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3941; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3942; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3943; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3944; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3945; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3946; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3947;
3948; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
3949; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3950; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3951; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3952; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3953; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3954; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3955; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3956; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3957; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3958; GFX90A-TGSPLIT-NEXT:    s_endpgm
3959;
3960; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
3961; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3962; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3963; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3964; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3965; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3966; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3967; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3968; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3969; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3970; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3971;
3972; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
3973; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3974; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3975; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3976; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3977; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3978; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3979; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3980; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3981; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3982; GFX940-TGSPLIT-NEXT:    s_endpgm
3983;
3984; GFX11-WGP-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
3985; GFX11-WGP:       ; %bb.0: ; %entry
3986; GFX11-WGP-NEXT:    s_clause 0x1
3987; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3988; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3989; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3990; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3991; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3992; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3993; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3994; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
3995; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3996; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
3997; GFX11-WGP-NEXT:    s_endpgm
3998;
3999; GFX11-CU-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
4000; GFX11-CU:       ; %bb.0: ; %entry
4001; GFX11-CU-NEXT:    s_clause 0x1
4002; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4003; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4004; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4005; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4006; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4007; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4008; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4009; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4010; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4011; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4012; GFX11-CU-NEXT:    s_endpgm
4013    i32* %out, i32 %in, i32 %old) {
4014entry:
4015  %gep = getelementptr i32, i32* %out, i32 4
4016  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic
4017  %val0 = extractvalue { i32, i1 } %val, 0
4018  store i32 %val0, i32* %out, align 4
4019  ret void
4020}
4021
4022define amdgpu_kernel void @flat_workgroup_acquire_monotonic_ret_cmpxchg(
4023; GFX7-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
4024; GFX7:       ; %bb.0: ; %entry
4025; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4026; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4027; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4028; GFX7-NEXT:    s_add_u32 s4, s0, 16
4029; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4030; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4031; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4032; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4033; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4034; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4035; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4036; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4037; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4038; GFX7-NEXT:    s_waitcnt vmcnt(0)
4039; GFX7-NEXT:    flat_store_dword v[0:1], v2
4040; GFX7-NEXT:    s_endpgm
4041;
4042; GFX10-WGP-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
4043; GFX10-WGP:       ; %bb.0: ; %entry
4044; GFX10-WGP-NEXT:    s_clause 0x1
4045; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4046; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4047; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4048; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4049; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4050; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4051; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4052; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4053; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4054; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4055; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4056; GFX10-WGP-NEXT:    buffer_gl0_inv
4057; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4058; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4059; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4060; GFX10-WGP-NEXT:    s_endpgm
4061;
4062; GFX10-CU-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
4063; GFX10-CU:       ; %bb.0: ; %entry
4064; GFX10-CU-NEXT:    s_clause 0x1
4065; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4066; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4067; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4068; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4069; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4070; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4071; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4072; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4073; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4074; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4075; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4076; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4077; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4078; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4079; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4080; GFX10-CU-NEXT:    s_endpgm
4081;
4082; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
4083; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4084; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4085; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4086; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4087; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4088; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4089; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4090; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4091; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4092; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4093; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4094; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4095; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4096; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4097; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4098; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4099; SKIP-CACHE-INV-NEXT:    s_endpgm
4100;
4101; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
4102; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4103; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4104; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4105; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4106; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4107; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4108; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4109; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4110; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4111; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4112;
4113; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
4114; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4115; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4116; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4117; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4118; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4119; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4120; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4121; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4122; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4123; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4124; GFX90A-TGSPLIT-NEXT:    s_endpgm
4125;
4126; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
4127; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4128; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4129; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4130; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4131; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4132; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4133; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4134; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4135; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4136; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4137;
4138; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
4139; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4140; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4141; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4142; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4143; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4144; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4145; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4146; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4147; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
4148; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4149; GFX940-TGSPLIT-NEXT:    s_endpgm
4150;
4151; GFX11-WGP-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
4152; GFX11-WGP:       ; %bb.0: ; %entry
4153; GFX11-WGP-NEXT:    s_clause 0x1
4154; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4155; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4156; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4157; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4158; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4159; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4160; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4161; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4162; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4163; GFX11-WGP-NEXT:    buffer_gl0_inv
4164; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4165; GFX11-WGP-NEXT:    s_endpgm
4166;
4167; GFX11-CU-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
4168; GFX11-CU:       ; %bb.0: ; %entry
4169; GFX11-CU-NEXT:    s_clause 0x1
4170; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4171; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4172; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4173; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4174; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4175; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4176; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4177; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4178; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4179; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4180; GFX11-CU-NEXT:    s_endpgm
4181    i32* %out, i32 %in, i32 %old) {
4182entry:
4183  %gep = getelementptr i32, i32* %out, i32 4
4184  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
4185  %val0 = extractvalue { i32, i1 } %val, 0
4186  store i32 %val0, i32* %out, align 4
4187  ret void
4188}
4189
4190define amdgpu_kernel void @flat_workgroup_release_monotonic_ret_cmpxchg(
4191; GFX7-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
4192; GFX7:       ; %bb.0: ; %entry
4193; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4194; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4195; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4196; GFX7-NEXT:    s_add_u32 s4, s0, 16
4197; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4198; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4199; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4200; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4201; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4202; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4203; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4204; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4205; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4206; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4207; GFX7-NEXT:    flat_store_dword v[0:1], v2
4208; GFX7-NEXT:    s_endpgm
4209;
4210; GFX10-WGP-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
4211; GFX10-WGP:       ; %bb.0: ; %entry
4212; GFX10-WGP-NEXT:    s_clause 0x1
4213; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4214; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4215; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4216; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4217; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4218; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4219; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4220; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4221; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4222; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4223; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4224; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4225; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4226; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4227; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4228; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4229; GFX10-WGP-NEXT:    s_endpgm
4230;
4231; GFX10-CU-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
4232; GFX10-CU:       ; %bb.0: ; %entry
4233; GFX10-CU-NEXT:    s_clause 0x1
4234; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4235; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4236; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4237; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4238; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4239; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4240; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4241; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4242; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4243; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4244; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4245; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4246; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4247; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4248; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4249; GFX10-CU-NEXT:    s_endpgm
4250;
4251; SKIP-CACHE-INV-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
4252; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4253; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4254; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4255; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4256; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4257; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4258; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4259; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4260; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4261; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4262; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4263; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4264; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4265; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4266; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4267; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4268; SKIP-CACHE-INV-NEXT:    s_endpgm
4269;
4270; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
4271; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4272; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4273; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4274; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4275; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4276; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4277; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4278; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4279; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4280; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4281; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4282;
4283; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
4284; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4285; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4286; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4287; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4288; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4289; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4290; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4291; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4292; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4293; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4294; GFX90A-TGSPLIT-NEXT:    s_endpgm
4295;
4296; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
4297; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4298; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4299; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4300; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4301; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4302; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4303; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4304; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4305; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4306; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4307; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4308;
4309; GFX940-TGSPLIT-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
4310; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4311; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4312; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4313; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4314; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4315; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4316; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4317; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4318; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4319; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4320; GFX940-TGSPLIT-NEXT:    s_endpgm
4321;
4322; GFX11-WGP-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
4323; GFX11-WGP:       ; %bb.0: ; %entry
4324; GFX11-WGP-NEXT:    s_clause 0x1
4325; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4326; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4327; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4328; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4329; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4330; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4331; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4332; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4333; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4334; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4335; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4336; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4337; GFX11-WGP-NEXT:    s_endpgm
4338;
4339; GFX11-CU-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
4340; GFX11-CU:       ; %bb.0: ; %entry
4341; GFX11-CU-NEXT:    s_clause 0x1
4342; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4343; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4344; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4345; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4346; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4347; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4348; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4349; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4350; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4351; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4352; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4353; GFX11-CU-NEXT:    s_endpgm
4354    i32* %out, i32 %in, i32 %old) {
4355entry:
4356  %gep = getelementptr i32, i32* %out, i32 4
4357  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic
4358  %val0 = extractvalue { i32, i1 } %val, 0
4359  store i32 %val0, i32* %out, align 4
4360  ret void
4361}
4362
4363define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_ret_cmpxchg(
4364; GFX7-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
4365; GFX7:       ; %bb.0: ; %entry
4366; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4367; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4368; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4369; GFX7-NEXT:    s_add_u32 s4, s0, 16
4370; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4371; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4372; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4373; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4374; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4375; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4376; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4377; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4378; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4379; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4380; GFX7-NEXT:    s_waitcnt vmcnt(0)
4381; GFX7-NEXT:    flat_store_dword v[0:1], v2
4382; GFX7-NEXT:    s_endpgm
4383;
4384; GFX10-WGP-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
4385; GFX10-WGP:       ; %bb.0: ; %entry
4386; GFX10-WGP-NEXT:    s_clause 0x1
4387; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4388; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4389; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4390; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4391; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4392; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4393; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4394; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4395; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4396; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4397; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4398; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4399; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4400; GFX10-WGP-NEXT:    buffer_gl0_inv
4401; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4402; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4403; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4404; GFX10-WGP-NEXT:    s_endpgm
4405;
4406; GFX10-CU-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
4407; GFX10-CU:       ; %bb.0: ; %entry
4408; GFX10-CU-NEXT:    s_clause 0x1
4409; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4410; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4411; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4412; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4413; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4414; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4415; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4416; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4417; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4418; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4419; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4420; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4421; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4422; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4423; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4424; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4425; GFX10-CU-NEXT:    s_endpgm
4426;
4427; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
4428; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4429; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4430; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4431; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4432; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4433; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4434; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4435; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4436; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4437; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4438; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4439; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4440; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4441; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4442; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4443; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4444; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4445; SKIP-CACHE-INV-NEXT:    s_endpgm
4446;
4447; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
4448; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4449; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4450; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4451; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4452; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4453; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4454; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4455; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4456; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4457; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4458; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4459;
4460; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
4461; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4462; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4463; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4464; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4465; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4466; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4467; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4468; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4469; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4470; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4471; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4472; GFX90A-TGSPLIT-NEXT:    s_endpgm
4473;
4474; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
4475; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4476; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4477; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4478; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4479; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4480; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4481; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4482; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4483; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4484; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4485; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4486;
4487; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
4488; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4489; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4490; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4491; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4492; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4493; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4494; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4495; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4496; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4497; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
4498; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4499; GFX940-TGSPLIT-NEXT:    s_endpgm
4500;
4501; GFX11-WGP-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
4502; GFX11-WGP:       ; %bb.0: ; %entry
4503; GFX11-WGP-NEXT:    s_clause 0x1
4504; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4505; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4506; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4507; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4508; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4509; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4510; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4511; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4512; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4513; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4514; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4515; GFX11-WGP-NEXT:    buffer_gl0_inv
4516; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4517; GFX11-WGP-NEXT:    s_endpgm
4518;
4519; GFX11-CU-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
4520; GFX11-CU:       ; %bb.0: ; %entry
4521; GFX11-CU-NEXT:    s_clause 0x1
4522; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4523; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4524; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4525; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4526; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4527; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4528; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4529; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4530; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4531; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4532; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4533; GFX11-CU-NEXT:    s_endpgm
4534    i32* %out, i32 %in, i32 %old) {
4535entry:
4536  %gep = getelementptr i32, i32* %out, i32 4
4537  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
4538  %val0 = extractvalue { i32, i1 } %val, 0
4539  store i32 %val0, i32* %out, align 4
4540  ret void
4541}
4542
4543define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_ret_cmpxchg(
4544; GFX7-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
4545; GFX7:       ; %bb.0: ; %entry
4546; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4547; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4548; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4549; GFX7-NEXT:    s_add_u32 s4, s0, 16
4550; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4551; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4552; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4553; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4554; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4555; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4556; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4557; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4558; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4559; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4560; GFX7-NEXT:    s_waitcnt vmcnt(0)
4561; GFX7-NEXT:    flat_store_dword v[0:1], v2
4562; GFX7-NEXT:    s_endpgm
4563;
4564; GFX10-WGP-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
4565; GFX10-WGP:       ; %bb.0: ; %entry
4566; GFX10-WGP-NEXT:    s_clause 0x1
4567; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4568; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4569; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4570; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4571; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4572; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4573; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4574; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4575; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4576; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4577; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4578; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4579; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4580; GFX10-WGP-NEXT:    buffer_gl0_inv
4581; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4582; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4583; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4584; GFX10-WGP-NEXT:    s_endpgm
4585;
4586; GFX10-CU-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
4587; GFX10-CU:       ; %bb.0: ; %entry
4588; GFX10-CU-NEXT:    s_clause 0x1
4589; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4590; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4591; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4592; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4593; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4594; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4595; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4596; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4597; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4598; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4599; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4600; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4601; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4602; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4603; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4604; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4605; GFX10-CU-NEXT:    s_endpgm
4606;
4607; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
4608; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4609; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4610; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4611; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4612; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4613; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4614; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4615; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4616; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4617; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4618; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4619; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4620; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4621; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4622; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4623; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4624; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4625; SKIP-CACHE-INV-NEXT:    s_endpgm
4626;
4627; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
4628; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4629; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4630; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4631; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4632; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4633; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4634; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4635; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4636; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4637; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4638; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4639;
4640; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
4641; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4642; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4643; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4644; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4645; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4646; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4647; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4648; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4649; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4650; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4651; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4652; GFX90A-TGSPLIT-NEXT:    s_endpgm
4653;
4654; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
4655; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4656; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4657; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4658; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4659; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4660; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4661; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4662; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4663; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4664; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4665; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4666;
4667; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
4668; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4669; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4670; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4671; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4672; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4673; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4674; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4675; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4676; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4677; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
4678; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4679; GFX940-TGSPLIT-NEXT:    s_endpgm
4680;
4681; GFX11-WGP-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
4682; GFX11-WGP:       ; %bb.0: ; %entry
4683; GFX11-WGP-NEXT:    s_clause 0x1
4684; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4685; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4686; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4687; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4688; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4689; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4690; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4691; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4692; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4693; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4694; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4695; GFX11-WGP-NEXT:    buffer_gl0_inv
4696; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4697; GFX11-WGP-NEXT:    s_endpgm
4698;
4699; GFX11-CU-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
4700; GFX11-CU:       ; %bb.0: ; %entry
4701; GFX11-CU-NEXT:    s_clause 0x1
4702; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4703; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4704; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4705; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4706; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4707; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4708; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4709; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4710; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4711; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4712; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4713; GFX11-CU-NEXT:    s_endpgm
4714    i32* %out, i32 %in, i32 %old) {
4715entry:
4716  %gep = getelementptr i32, i32* %out, i32 4
4717  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
4718  %val0 = extractvalue { i32, i1 } %val, 0
4719  store i32 %val0, i32* %out, align 4
4720  ret void
4721}
4722
4723define amdgpu_kernel void @flat_workgroup_monotonic_acquire_ret_cmpxchg(
4724; GFX7-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
4725; GFX7:       ; %bb.0: ; %entry
4726; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4727; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4728; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4729; GFX7-NEXT:    s_add_u32 s4, s0, 16
4730; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4731; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4732; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4733; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4734; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4735; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4736; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4737; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4738; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4739; GFX7-NEXT:    s_waitcnt vmcnt(0)
4740; GFX7-NEXT:    flat_store_dword v[0:1], v2
4741; GFX7-NEXT:    s_endpgm
4742;
4743; GFX10-WGP-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
4744; GFX10-WGP:       ; %bb.0: ; %entry
4745; GFX10-WGP-NEXT:    s_clause 0x1
4746; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4747; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4748; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4749; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4750; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4751; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4752; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4753; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4754; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4755; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4756; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4757; GFX10-WGP-NEXT:    buffer_gl0_inv
4758; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4759; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4760; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4761; GFX10-WGP-NEXT:    s_endpgm
4762;
4763; GFX10-CU-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
4764; GFX10-CU:       ; %bb.0: ; %entry
4765; GFX10-CU-NEXT:    s_clause 0x1
4766; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4767; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4768; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4769; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4770; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4771; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4772; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4773; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4774; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4775; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4776; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4777; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4778; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4779; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4780; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4781; GFX10-CU-NEXT:    s_endpgm
4782;
4783; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
4784; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4785; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4786; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4787; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4788; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4789; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4790; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4791; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4792; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4793; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4794; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4795; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4796; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4797; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4798; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4799; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4800; SKIP-CACHE-INV-NEXT:    s_endpgm
4801;
4802; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
4803; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4804; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4805; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4806; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4807; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4808; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4809; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4810; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4811; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4812; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4813;
4814; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
4815; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4816; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4817; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4818; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4819; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4820; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4821; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4822; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4823; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4824; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4825; GFX90A-TGSPLIT-NEXT:    s_endpgm
4826;
4827; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
4828; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4829; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4830; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4831; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4832; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4833; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4834; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4835; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4836; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4837; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4838;
4839; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
4840; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4841; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4842; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4843; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4844; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4845; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4846; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4847; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4848; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
4849; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4850; GFX940-TGSPLIT-NEXT:    s_endpgm
4851;
4852; GFX11-WGP-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
4853; GFX11-WGP:       ; %bb.0: ; %entry
4854; GFX11-WGP-NEXT:    s_clause 0x1
4855; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4856; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4857; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4858; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4859; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4860; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4861; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4862; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4863; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4864; GFX11-WGP-NEXT:    buffer_gl0_inv
4865; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4866; GFX11-WGP-NEXT:    s_endpgm
4867;
4868; GFX11-CU-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
4869; GFX11-CU:       ; %bb.0: ; %entry
4870; GFX11-CU-NEXT:    s_clause 0x1
4871; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4872; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4873; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4874; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4875; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4876; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4877; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4878; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4879; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4880; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4881; GFX11-CU-NEXT:    s_endpgm
4882    i32* %out, i32 %in, i32 %old) {
4883entry:
4884  %gep = getelementptr i32, i32* %out, i32 4
4885  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic acquire
4886  %val0 = extractvalue { i32, i1 } %val, 0
4887  store i32 %val0, i32* %out, align 4
4888  ret void
4889}
4890
4891define amdgpu_kernel void @flat_workgroup_acquire_acquire_ret_cmpxchg(
4892; GFX7-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
4893; GFX7:       ; %bb.0: ; %entry
4894; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4895; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4896; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4897; GFX7-NEXT:    s_add_u32 s4, s0, 16
4898; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4899; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4900; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4901; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4902; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4903; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4904; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4905; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4906; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4907; GFX7-NEXT:    s_waitcnt vmcnt(0)
4908; GFX7-NEXT:    flat_store_dword v[0:1], v2
4909; GFX7-NEXT:    s_endpgm
4910;
4911; GFX10-WGP-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
4912; GFX10-WGP:       ; %bb.0: ; %entry
4913; GFX10-WGP-NEXT:    s_clause 0x1
4914; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4915; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4916; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4917; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4918; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4919; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4920; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4921; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4922; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4923; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4924; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4925; GFX10-WGP-NEXT:    buffer_gl0_inv
4926; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4927; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4928; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4929; GFX10-WGP-NEXT:    s_endpgm
4930;
4931; GFX10-CU-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
4932; GFX10-CU:       ; %bb.0: ; %entry
4933; GFX10-CU-NEXT:    s_clause 0x1
4934; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4935; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4936; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4937; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4938; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4939; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4940; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4941; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4942; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4943; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4944; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4945; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4946; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4947; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4948; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4949; GFX10-CU-NEXT:    s_endpgm
4950;
4951; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
4952; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4953; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4954; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4955; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4956; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4957; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4958; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4959; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4960; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4961; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4962; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4963; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4964; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4965; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4966; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4967; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4968; SKIP-CACHE-INV-NEXT:    s_endpgm
4969;
4970; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
4971; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4972; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4973; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4974; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4975; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4976; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4977; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4978; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4979; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4980; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4981;
4982; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
4983; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4984; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4985; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4986; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4987; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4988; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4989; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4990; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4991; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4992; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4993; GFX90A-TGSPLIT-NEXT:    s_endpgm
4994;
4995; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
4996; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4997; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4998; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4999; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5000; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5001; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5002; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5003; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5004; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5005; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5006;
5007; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
5008; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5009; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5010; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5011; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5012; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5013; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5014; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5015; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5016; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
5017; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5018; GFX940-TGSPLIT-NEXT:    s_endpgm
5019;
5020; GFX11-WGP-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
5021; GFX11-WGP:       ; %bb.0: ; %entry
5022; GFX11-WGP-NEXT:    s_clause 0x1
5023; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5024; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5025; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5026; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5027; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5028; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5029; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5030; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5031; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5032; GFX11-WGP-NEXT:    buffer_gl0_inv
5033; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5034; GFX11-WGP-NEXT:    s_endpgm
5035;
5036; GFX11-CU-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
5037; GFX11-CU:       ; %bb.0: ; %entry
5038; GFX11-CU-NEXT:    s_clause 0x1
5039; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5040; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5041; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5042; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5043; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5044; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5045; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5046; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5047; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5048; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5049; GFX11-CU-NEXT:    s_endpgm
5050    i32* %out, i32 %in, i32 %old) {
5051entry:
5052  %gep = getelementptr i32, i32* %out, i32 4
5053  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
5054  %val0 = extractvalue { i32, i1 } %val, 0
5055  store i32 %val0, i32* %out, align 4
5056  ret void
5057}
5058
5059define amdgpu_kernel void @flat_workgroup_release_acquire_ret_cmpxchg(
5060; GFX7-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
5061; GFX7:       ; %bb.0: ; %entry
5062; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5063; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5064; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5065; GFX7-NEXT:    s_add_u32 s4, s0, 16
5066; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5067; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5068; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5069; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5070; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5071; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5072; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5073; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5074; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5075; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5076; GFX7-NEXT:    s_waitcnt vmcnt(0)
5077; GFX7-NEXT:    flat_store_dword v[0:1], v2
5078; GFX7-NEXT:    s_endpgm
5079;
5080; GFX10-WGP-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
5081; GFX10-WGP:       ; %bb.0: ; %entry
5082; GFX10-WGP-NEXT:    s_clause 0x1
5083; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5084; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5085; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5086; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5087; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5088; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5089; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5090; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5091; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5092; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5093; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5094; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5095; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5096; GFX10-WGP-NEXT:    buffer_gl0_inv
5097; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5098; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5099; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5100; GFX10-WGP-NEXT:    s_endpgm
5101;
5102; GFX10-CU-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
5103; GFX10-CU:       ; %bb.0: ; %entry
5104; GFX10-CU-NEXT:    s_clause 0x1
5105; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5106; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5107; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5108; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5109; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5110; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5111; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5112; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5113; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5114; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5115; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5116; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5117; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5118; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5119; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5120; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5121; GFX10-CU-NEXT:    s_endpgm
5122;
5123; SKIP-CACHE-INV-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
5124; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5125; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5126; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5127; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5128; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5129; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5130; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5131; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5132; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5133; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5134; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5135; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5136; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5137; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5138; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5139; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5140; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5141; SKIP-CACHE-INV-NEXT:    s_endpgm
5142;
5143; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
5144; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5145; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5146; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5147; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5148; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5149; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5150; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5151; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5152; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5153; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5154; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5155;
5156; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
5157; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5158; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5159; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5160; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5161; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5162; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5163; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5164; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5165; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5166; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5167; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5168; GFX90A-TGSPLIT-NEXT:    s_endpgm
5169;
5170; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
5171; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5172; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5173; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5174; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5175; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5176; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5177; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5178; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5179; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5180; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5181; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5182;
5183; GFX940-TGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
5184; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5185; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5186; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5187; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5188; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5189; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5190; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5191; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5192; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5193; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
5194; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5195; GFX940-TGSPLIT-NEXT:    s_endpgm
5196;
5197; GFX11-WGP-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
5198; GFX11-WGP:       ; %bb.0: ; %entry
5199; GFX11-WGP-NEXT:    s_clause 0x1
5200; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5201; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5202; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5203; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5204; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5205; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5206; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5207; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5208; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5209; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5210; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5211; GFX11-WGP-NEXT:    buffer_gl0_inv
5212; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5213; GFX11-WGP-NEXT:    s_endpgm
5214;
5215; GFX11-CU-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
5216; GFX11-CU:       ; %bb.0: ; %entry
5217; GFX11-CU-NEXT:    s_clause 0x1
5218; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5219; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5220; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5221; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5222; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5223; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5224; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5225; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5226; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5227; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5228; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5229; GFX11-CU-NEXT:    s_endpgm
5230    i32* %out, i32 %in, i32 %old) {
5231entry:
5232  %gep = getelementptr i32, i32* %out, i32 4
5233  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
5234  %val0 = extractvalue { i32, i1 } %val, 0
5235  store i32 %val0, i32* %out, align 4
5236  ret void
5237}
5238
5239define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_ret_cmpxchg(
5240; GFX7-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
5241; GFX7:       ; %bb.0: ; %entry
5242; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5243; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5244; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5245; GFX7-NEXT:    s_add_u32 s4, s0, 16
5246; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5247; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5248; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5249; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5250; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5251; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5252; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5253; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5254; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5255; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5256; GFX7-NEXT:    s_waitcnt vmcnt(0)
5257; GFX7-NEXT:    flat_store_dword v[0:1], v2
5258; GFX7-NEXT:    s_endpgm
5259;
5260; GFX10-WGP-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
5261; GFX10-WGP:       ; %bb.0: ; %entry
5262; GFX10-WGP-NEXT:    s_clause 0x1
5263; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5264; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5265; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5266; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5267; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5268; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5269; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5270; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5271; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5272; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5273; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5274; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5275; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5276; GFX10-WGP-NEXT:    buffer_gl0_inv
5277; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5278; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5279; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5280; GFX10-WGP-NEXT:    s_endpgm
5281;
5282; GFX10-CU-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
5283; GFX10-CU:       ; %bb.0: ; %entry
5284; GFX10-CU-NEXT:    s_clause 0x1
5285; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5286; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5287; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5288; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5289; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5290; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5291; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5292; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5293; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5294; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5295; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5296; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5297; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5298; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5299; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5300; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5301; GFX10-CU-NEXT:    s_endpgm
5302;
5303; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
5304; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5305; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5306; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5307; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5308; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5309; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5310; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5311; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5312; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5313; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5314; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5315; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5316; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5317; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5318; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5319; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5320; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5321; SKIP-CACHE-INV-NEXT:    s_endpgm
5322;
5323; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
5324; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5325; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5326; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5327; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5328; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5329; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5330; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5331; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5332; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5333; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5334; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5335;
5336; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
5337; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5338; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5339; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5340; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5341; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5342; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5343; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5344; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5345; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5346; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5347; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5348; GFX90A-TGSPLIT-NEXT:    s_endpgm
5349;
5350; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
5351; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5352; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5353; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5354; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5355; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5356; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5357; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5358; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5359; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5360; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5361; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5362;
5363; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
5364; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5365; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5366; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5367; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5368; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5369; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5370; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5371; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5372; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5373; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
5374; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5375; GFX940-TGSPLIT-NEXT:    s_endpgm
5376;
5377; GFX11-WGP-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
5378; GFX11-WGP:       ; %bb.0: ; %entry
5379; GFX11-WGP-NEXT:    s_clause 0x1
5380; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5381; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5382; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5383; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5384; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5385; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5386; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5387; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5388; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5389; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5390; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5391; GFX11-WGP-NEXT:    buffer_gl0_inv
5392; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5393; GFX11-WGP-NEXT:    s_endpgm
5394;
5395; GFX11-CU-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
5396; GFX11-CU:       ; %bb.0: ; %entry
5397; GFX11-CU-NEXT:    s_clause 0x1
5398; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5399; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5400; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5401; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5402; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5403; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5404; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5405; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5406; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5407; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5408; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5409; GFX11-CU-NEXT:    s_endpgm
5410    i32* %out, i32 %in, i32 %old) {
5411entry:
5412  %gep = getelementptr i32, i32* %out, i32 4
5413  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
5414  %val0 = extractvalue { i32, i1 } %val, 0
5415  store i32 %val0, i32* %out, align 4
5416  ret void
5417}
5418
5419define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_ret_cmpxchg(
5420; GFX7-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
5421; GFX7:       ; %bb.0: ; %entry
5422; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5423; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5424; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5425; GFX7-NEXT:    s_add_u32 s4, s0, 16
5426; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5427; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5428; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5429; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5430; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5431; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5432; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5433; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5434; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5435; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5436; GFX7-NEXT:    s_waitcnt vmcnt(0)
5437; GFX7-NEXT:    flat_store_dword v[0:1], v2
5438; GFX7-NEXT:    s_endpgm
5439;
5440; GFX10-WGP-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
5441; GFX10-WGP:       ; %bb.0: ; %entry
5442; GFX10-WGP-NEXT:    s_clause 0x1
5443; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5444; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5445; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5446; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5447; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5448; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5449; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5450; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5451; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5452; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5453; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5454; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5455; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5456; GFX10-WGP-NEXT:    buffer_gl0_inv
5457; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5458; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5459; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5460; GFX10-WGP-NEXT:    s_endpgm
5461;
5462; GFX10-CU-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
5463; GFX10-CU:       ; %bb.0: ; %entry
5464; GFX10-CU-NEXT:    s_clause 0x1
5465; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5466; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5467; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5468; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5469; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5470; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5471; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5472; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5473; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5474; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5475; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5476; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5477; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5478; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5479; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5480; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5481; GFX10-CU-NEXT:    s_endpgm
5482;
5483; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
5484; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5485; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5486; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5487; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5488; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5489; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5490; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5491; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5492; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5493; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5494; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5495; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5496; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5497; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5498; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5499; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5500; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5501; SKIP-CACHE-INV-NEXT:    s_endpgm
5502;
5503; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
5504; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5505; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5506; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5507; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5508; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5509; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5510; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5511; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5512; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5513; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5514; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5515;
5516; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
5517; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5518; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5519; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5520; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5521; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5522; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5523; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5524; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5525; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5526; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5527; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5528; GFX90A-TGSPLIT-NEXT:    s_endpgm
5529;
5530; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
5531; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5532; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5533; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5534; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5535; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5536; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5537; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5538; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5539; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5540; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5541; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5542;
5543; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
5544; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5545; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5546; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5547; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5548; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5549; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5550; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5551; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5552; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5553; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
5554; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5555; GFX940-TGSPLIT-NEXT:    s_endpgm
5556;
5557; GFX11-WGP-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
5558; GFX11-WGP:       ; %bb.0: ; %entry
5559; GFX11-WGP-NEXT:    s_clause 0x1
5560; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5561; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5562; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5563; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5564; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5565; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5566; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5567; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5568; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5569; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5570; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5571; GFX11-WGP-NEXT:    buffer_gl0_inv
5572; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5573; GFX11-WGP-NEXT:    s_endpgm
5574;
5575; GFX11-CU-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
5576; GFX11-CU:       ; %bb.0: ; %entry
5577; GFX11-CU-NEXT:    s_clause 0x1
5578; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5579; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5580; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5581; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5582; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5583; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5584; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5585; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5586; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5587; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5588; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5589; GFX11-CU-NEXT:    s_endpgm
5590    i32* %out, i32 %in, i32 %old) {
5591entry:
5592  %gep = getelementptr i32, i32* %out, i32 4
5593  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
5594  %val0 = extractvalue { i32, i1 } %val, 0
5595  store i32 %val0, i32* %out, align 4
5596  ret void
5597}
5598
5599define amdgpu_kernel void @flat_workgroup_monotonic_seq_cst_ret_cmpxchg(
5600; GFX7-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
5601; GFX7:       ; %bb.0: ; %entry
5602; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5603; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5604; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5605; GFX7-NEXT:    s_add_u32 s4, s0, 16
5606; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5607; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5608; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5609; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5610; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5611; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5612; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5613; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5614; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5615; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5616; GFX7-NEXT:    s_waitcnt vmcnt(0)
5617; GFX7-NEXT:    flat_store_dword v[0:1], v2
5618; GFX7-NEXT:    s_endpgm
5619;
5620; GFX10-WGP-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
5621; GFX10-WGP:       ; %bb.0: ; %entry
5622; GFX10-WGP-NEXT:    s_clause 0x1
5623; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5624; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5625; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5626; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5627; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5628; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5629; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5630; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5631; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5632; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5633; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5634; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5635; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5636; GFX10-WGP-NEXT:    buffer_gl0_inv
5637; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5638; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5639; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5640; GFX10-WGP-NEXT:    s_endpgm
5641;
5642; GFX10-CU-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
5643; GFX10-CU:       ; %bb.0: ; %entry
5644; GFX10-CU-NEXT:    s_clause 0x1
5645; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5646; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5647; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5648; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5649; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5650; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5651; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5652; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5653; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5654; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5655; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5656; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5657; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5658; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5659; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5660; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5661; GFX10-CU-NEXT:    s_endpgm
5662;
5663; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
5664; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5665; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5666; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5667; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5668; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5669; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5670; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5671; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5672; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5673; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5674; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5675; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5676; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5677; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5678; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5679; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5680; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5681; SKIP-CACHE-INV-NEXT:    s_endpgm
5682;
5683; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
5684; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5685; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5686; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5687; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5688; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5689; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5690; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5691; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5692; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5693; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5694; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5695;
5696; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
5697; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5698; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5699; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5700; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5701; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5702; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5703; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5704; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5705; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5706; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5707; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5708; GFX90A-TGSPLIT-NEXT:    s_endpgm
5709;
5710; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
5711; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5712; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5713; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5714; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5715; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5716; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5717; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5718; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5719; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5720; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5721; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5722;
5723; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
5724; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5725; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5726; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5727; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5728; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5729; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5730; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5731; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5732; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5733; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
5734; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5735; GFX940-TGSPLIT-NEXT:    s_endpgm
5736;
5737; GFX11-WGP-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
5738; GFX11-WGP:       ; %bb.0: ; %entry
5739; GFX11-WGP-NEXT:    s_clause 0x1
5740; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5741; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5742; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5743; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5744; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5745; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5746; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5747; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5748; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5749; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5750; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5751; GFX11-WGP-NEXT:    buffer_gl0_inv
5752; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5753; GFX11-WGP-NEXT:    s_endpgm
5754;
5755; GFX11-CU-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
5756; GFX11-CU:       ; %bb.0: ; %entry
5757; GFX11-CU-NEXT:    s_clause 0x1
5758; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5759; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5760; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5761; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5762; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5763; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5764; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5765; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5766; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5767; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5768; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5769; GFX11-CU-NEXT:    s_endpgm
5770    i32* %out, i32 %in, i32 %old) {
5771entry:
5772  %gep = getelementptr i32, i32* %out, i32 4
5773  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic seq_cst
5774  %val0 = extractvalue { i32, i1 } %val, 0
5775  store i32 %val0, i32* %out, align 4
5776  ret void
5777}
5778
5779define amdgpu_kernel void @flat_workgroup_acquire_seq_cst_ret_cmpxchg(
5780; GFX7-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
5781; GFX7:       ; %bb.0: ; %entry
5782; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5783; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5784; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5785; GFX7-NEXT:    s_add_u32 s4, s0, 16
5786; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5787; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5788; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5789; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5790; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5791; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5792; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5793; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5794; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5795; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5796; GFX7-NEXT:    s_waitcnt vmcnt(0)
5797; GFX7-NEXT:    flat_store_dword v[0:1], v2
5798; GFX7-NEXT:    s_endpgm
5799;
5800; GFX10-WGP-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
5801; GFX10-WGP:       ; %bb.0: ; %entry
5802; GFX10-WGP-NEXT:    s_clause 0x1
5803; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5804; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5805; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5806; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5807; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5808; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5809; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5810; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5811; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5812; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5813; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5814; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5815; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5816; GFX10-WGP-NEXT:    buffer_gl0_inv
5817; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5818; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5819; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5820; GFX10-WGP-NEXT:    s_endpgm
5821;
5822; GFX10-CU-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
5823; GFX10-CU:       ; %bb.0: ; %entry
5824; GFX10-CU-NEXT:    s_clause 0x1
5825; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5826; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5827; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5828; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5829; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5830; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5831; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5832; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5833; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5834; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5835; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5836; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5837; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5838; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5839; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5840; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5841; GFX10-CU-NEXT:    s_endpgm
5842;
5843; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
5844; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5845; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5846; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5847; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5848; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5849; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5850; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5851; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5852; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5853; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5854; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5855; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5856; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5857; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5858; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5859; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5860; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5861; SKIP-CACHE-INV-NEXT:    s_endpgm
5862;
5863; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
5864; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5865; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5866; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5867; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5868; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5869; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5870; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5871; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5872; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5873; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5874; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5875;
5876; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
5877; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5878; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5879; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5880; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5881; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5882; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5883; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5884; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5885; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5886; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5887; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5888; GFX90A-TGSPLIT-NEXT:    s_endpgm
5889;
5890; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
5891; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5892; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5893; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5894; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5895; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5896; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5897; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5898; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5899; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5900; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5901; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5902;
5903; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
5904; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5905; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5906; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5907; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5908; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5909; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5910; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5911; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5912; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5913; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
5914; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5915; GFX940-TGSPLIT-NEXT:    s_endpgm
5916;
5917; GFX11-WGP-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
5918; GFX11-WGP:       ; %bb.0: ; %entry
5919; GFX11-WGP-NEXT:    s_clause 0x1
5920; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5921; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5922; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5923; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5924; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5925; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5926; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5927; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5928; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5929; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5930; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5931; GFX11-WGP-NEXT:    buffer_gl0_inv
5932; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5933; GFX11-WGP-NEXT:    s_endpgm
5934;
5935; GFX11-CU-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
5936; GFX11-CU:       ; %bb.0: ; %entry
5937; GFX11-CU-NEXT:    s_clause 0x1
5938; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5939; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5940; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5941; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5942; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5943; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5944; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5945; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5946; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5947; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5948; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5949; GFX11-CU-NEXT:    s_endpgm
5950    i32* %out, i32 %in, i32 %old) {
5951entry:
5952  %gep = getelementptr i32, i32* %out, i32 4
5953  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire seq_cst
5954  %val0 = extractvalue { i32, i1 } %val, 0
5955  store i32 %val0, i32* %out, align 4
5956  ret void
5957}
5958
5959define amdgpu_kernel void @flat_workgroup_release_seq_cst_ret_cmpxchg(
5960; GFX7-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
5961; GFX7:       ; %bb.0: ; %entry
5962; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5963; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5964; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5965; GFX7-NEXT:    s_add_u32 s4, s0, 16
5966; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5967; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5968; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5969; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5970; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5971; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5972; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5973; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5974; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5975; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5976; GFX7-NEXT:    s_waitcnt vmcnt(0)
5977; GFX7-NEXT:    flat_store_dword v[0:1], v2
5978; GFX7-NEXT:    s_endpgm
5979;
5980; GFX10-WGP-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
5981; GFX10-WGP:       ; %bb.0: ; %entry
5982; GFX10-WGP-NEXT:    s_clause 0x1
5983; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5984; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5985; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5986; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5987; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5988; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5989; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5990; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5991; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5992; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5993; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5994; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5995; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5996; GFX10-WGP-NEXT:    buffer_gl0_inv
5997; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5998; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5999; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6000; GFX10-WGP-NEXT:    s_endpgm
6001;
6002; GFX10-CU-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
6003; GFX10-CU:       ; %bb.0: ; %entry
6004; GFX10-CU-NEXT:    s_clause 0x1
6005; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6006; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6007; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6008; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6009; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6010; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6011; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6012; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6013; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6014; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6015; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6016; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6017; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6018; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6019; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6020; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6021; GFX10-CU-NEXT:    s_endpgm
6022;
6023; SKIP-CACHE-INV-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
6024; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6025; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6026; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6027; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6028; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6029; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6030; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6031; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6032; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6033; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6034; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6035; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6036; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6037; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6038; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6039; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6040; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6041; SKIP-CACHE-INV-NEXT:    s_endpgm
6042;
6043; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
6044; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6045; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6046; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6047; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6048; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6049; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6050; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6051; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6052; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6053; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6054; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6055;
6056; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
6057; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6058; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6059; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6060; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6061; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6062; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6063; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6064; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6065; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6066; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6067; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6068; GFX90A-TGSPLIT-NEXT:    s_endpgm
6069;
6070; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
6071; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6072; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6073; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6074; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6075; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6076; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6077; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6078; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6079; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6080; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6081; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6082;
6083; GFX940-TGSPLIT-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
6084; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6085; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6086; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6087; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6088; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6089; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6090; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6091; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6092; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6093; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
6094; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6095; GFX940-TGSPLIT-NEXT:    s_endpgm
6096;
6097; GFX11-WGP-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
6098; GFX11-WGP:       ; %bb.0: ; %entry
6099; GFX11-WGP-NEXT:    s_clause 0x1
6100; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6101; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6102; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6103; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6104; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
6105; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6106; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
6107; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6108; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6109; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6110; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6111; GFX11-WGP-NEXT:    buffer_gl0_inv
6112; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6113; GFX11-WGP-NEXT:    s_endpgm
6114;
6115; GFX11-CU-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
6116; GFX11-CU:       ; %bb.0: ; %entry
6117; GFX11-CU-NEXT:    s_clause 0x1
6118; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6119; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6120; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6121; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6122; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
6123; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6124; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6125; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6126; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6127; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6128; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6129; GFX11-CU-NEXT:    s_endpgm
6130    i32* %out, i32 %in, i32 %old) {
6131entry:
6132  %gep = getelementptr i32, i32* %out, i32 4
6133  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release seq_cst
6134  %val0 = extractvalue { i32, i1 } %val, 0
6135  store i32 %val0, i32* %out, align 4
6136  ret void
6137}
6138
6139define amdgpu_kernel void @flat_workgroup_acq_rel_seq_cst_ret_cmpxchg(
6140; GFX7-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
6141; GFX7:       ; %bb.0: ; %entry
6142; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6143; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6144; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6145; GFX7-NEXT:    s_add_u32 s4, s0, 16
6146; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6147; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6148; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6149; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6150; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6151; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6152; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6153; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6154; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6155; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6156; GFX7-NEXT:    s_waitcnt vmcnt(0)
6157; GFX7-NEXT:    flat_store_dword v[0:1], v2
6158; GFX7-NEXT:    s_endpgm
6159;
6160; GFX10-WGP-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
6161; GFX10-WGP:       ; %bb.0: ; %entry
6162; GFX10-WGP-NEXT:    s_clause 0x1
6163; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6164; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6165; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6166; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6167; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6168; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6169; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6170; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6171; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6172; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6173; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6174; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6175; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6176; GFX10-WGP-NEXT:    buffer_gl0_inv
6177; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6178; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6179; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6180; GFX10-WGP-NEXT:    s_endpgm
6181;
6182; GFX10-CU-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
6183; GFX10-CU:       ; %bb.0: ; %entry
6184; GFX10-CU-NEXT:    s_clause 0x1
6185; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6186; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6187; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6188; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6189; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6190; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6191; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6192; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6193; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6194; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6195; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6196; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6197; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6198; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6199; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6200; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6201; GFX10-CU-NEXT:    s_endpgm
6202;
6203; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
6204; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6205; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6206; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6207; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6208; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6209; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6210; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6211; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6212; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6213; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6214; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6215; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6216; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6217; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6218; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6219; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6220; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6221; SKIP-CACHE-INV-NEXT:    s_endpgm
6222;
6223; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
6224; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6225; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6226; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6227; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6228; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6229; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6230; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6231; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6232; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6233; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6234; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6235;
6236; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
6237; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6238; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6239; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6240; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6241; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6242; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6243; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6244; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6245; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6246; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6247; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6248; GFX90A-TGSPLIT-NEXT:    s_endpgm
6249;
6250; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
6251; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6252; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6253; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6254; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6255; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6256; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6257; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6258; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6259; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6260; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6261; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6262;
6263; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
6264; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6265; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6266; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6267; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6268; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6269; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6270; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6271; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6272; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6273; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
6274; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6275; GFX940-TGSPLIT-NEXT:    s_endpgm
6276;
6277; GFX11-WGP-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
6278; GFX11-WGP:       ; %bb.0: ; %entry
6279; GFX11-WGP-NEXT:    s_clause 0x1
6280; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6281; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6282; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6283; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6284; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
6285; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6286; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
6287; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6288; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6289; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6290; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6291; GFX11-WGP-NEXT:    buffer_gl0_inv
6292; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6293; GFX11-WGP-NEXT:    s_endpgm
6294;
6295; GFX11-CU-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
6296; GFX11-CU:       ; %bb.0: ; %entry
6297; GFX11-CU-NEXT:    s_clause 0x1
6298; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6299; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6300; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6301; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6302; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
6303; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6304; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6305; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6306; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6307; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6308; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6309; GFX11-CU-NEXT:    s_endpgm
6310    i32* %out, i32 %in, i32 %old) {
6311entry:
6312  %gep = getelementptr i32, i32* %out, i32 4
6313  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel seq_cst
6314  %val0 = extractvalue { i32, i1 } %val, 0
6315  store i32 %val0, i32* %out, align 4
6316  ret void
6317}
6318
6319define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_ret_cmpxchg(
6320; GFX7-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
6321; GFX7:       ; %bb.0: ; %entry
6322; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6323; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6324; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6325; GFX7-NEXT:    s_add_u32 s4, s0, 16
6326; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6327; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6328; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6329; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6330; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6331; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6332; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6333; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6334; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6335; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6336; GFX7-NEXT:    s_waitcnt vmcnt(0)
6337; GFX7-NEXT:    flat_store_dword v[0:1], v2
6338; GFX7-NEXT:    s_endpgm
6339;
6340; GFX10-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
6341; GFX10-WGP:       ; %bb.0: ; %entry
6342; GFX10-WGP-NEXT:    s_clause 0x1
6343; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6344; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6345; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6346; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6347; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6348; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6349; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6350; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6351; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6352; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6353; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6354; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6355; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6356; GFX10-WGP-NEXT:    buffer_gl0_inv
6357; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6358; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6359; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6360; GFX10-WGP-NEXT:    s_endpgm
6361;
6362; GFX10-CU-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
6363; GFX10-CU:       ; %bb.0: ; %entry
6364; GFX10-CU-NEXT:    s_clause 0x1
6365; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6366; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6367; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6368; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6369; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6370; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6371; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6372; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6373; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6374; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6375; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6376; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6377; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6378; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6379; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6380; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6381; GFX10-CU-NEXT:    s_endpgm
6382;
6383; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
6384; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6385; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6386; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6387; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6388; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6389; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6390; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6391; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6392; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6393; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6394; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6395; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6396; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6397; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6398; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6399; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6400; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6401; SKIP-CACHE-INV-NEXT:    s_endpgm
6402;
6403; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
6404; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6405; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6406; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6407; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6408; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6409; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6410; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6411; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6412; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6413; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6414; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6415;
6416; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
6417; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6418; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6419; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6420; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6421; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6422; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6423; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6424; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6425; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6426; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6427; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6428; GFX90A-TGSPLIT-NEXT:    s_endpgm
6429;
6430; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
6431; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6432; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6433; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6434; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6435; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6436; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6437; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6438; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6439; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6440; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6441; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6442;
6443; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
6444; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6445; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6446; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6447; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6448; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6449; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6450; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6451; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6452; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6453; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
6454; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6455; GFX940-TGSPLIT-NEXT:    s_endpgm
6456;
6457; GFX11-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
6458; GFX11-WGP:       ; %bb.0: ; %entry
6459; GFX11-WGP-NEXT:    s_clause 0x1
6460; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6461; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6462; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6463; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6464; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
6465; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6466; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
6467; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6468; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6469; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6470; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6471; GFX11-WGP-NEXT:    buffer_gl0_inv
6472; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6473; GFX11-WGP-NEXT:    s_endpgm
6474;
6475; GFX11-CU-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
6476; GFX11-CU:       ; %bb.0: ; %entry
6477; GFX11-CU-NEXT:    s_clause 0x1
6478; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6479; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6480; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6481; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6482; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
6483; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6484; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6485; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6486; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6487; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6488; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6489; GFX11-CU-NEXT:    s_endpgm
6490    i32* %out, i32 %in, i32 %old) {
6491entry:
6492  %gep = getelementptr i32, i32* %out, i32 4
6493  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
6494  %val0 = extractvalue { i32, i1 } %val, 0
6495  store i32 %val0, i32* %out, align 4
6496  ret void
6497}
6498
6499define amdgpu_kernel void @flat_workgroup_one_as_unordered_load(
6500; GFX7-LABEL: flat_workgroup_one_as_unordered_load:
6501; GFX7:       ; %bb.0: ; %entry
6502; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6503; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6504; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6505; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6506; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6507; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6508; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6509; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6510; GFX7-NEXT:    flat_store_dword v[0:1], v2
6511; GFX7-NEXT:    s_endpgm
6512;
6513; GFX10-WGP-LABEL: flat_workgroup_one_as_unordered_load:
6514; GFX10-WGP:       ; %bb.0: ; %entry
6515; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6516; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6517; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6518; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6519; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
6520; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6521; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6522; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6523; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6524; GFX10-WGP-NEXT:    s_endpgm
6525;
6526; GFX10-CU-LABEL: flat_workgroup_one_as_unordered_load:
6527; GFX10-CU:       ; %bb.0: ; %entry
6528; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6529; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6530; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6531; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6532; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6533; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6534; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6535; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6536; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6537; GFX10-CU-NEXT:    s_endpgm
6538;
6539; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_unordered_load:
6540; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6541; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6542; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6543; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6544; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6545; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
6546; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6547; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6548; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6549; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6550; SKIP-CACHE-INV-NEXT:    s_endpgm
6551;
6552; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_load:
6553; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6554; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6555; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6556; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6557; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6558; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6559; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6560; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6561; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6562; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6563; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6564;
6565; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_load:
6566; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6567; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6568; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6569; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6570; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6571; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6572; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6573; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6574; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6575; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6576; GFX90A-TGSPLIT-NEXT:    s_endpgm
6577;
6578; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_load:
6579; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6580; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6581; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6582; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6583; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6584; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6585; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6586; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6587; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6588; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6589; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6590;
6591; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_load:
6592; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6593; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6594; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6595; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6596; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6597; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6598; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6599; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6600; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6601; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6602; GFX940-TGSPLIT-NEXT:    s_endpgm
6603;
6604; GFX11-WGP-LABEL: flat_workgroup_one_as_unordered_load:
6605; GFX11-WGP:       ; %bb.0: ; %entry
6606; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6607; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6608; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
6609; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
6610; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
6611; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6612; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6613; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6614; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6615; GFX11-WGP-NEXT:    s_endpgm
6616;
6617; GFX11-CU-LABEL: flat_workgroup_one_as_unordered_load:
6618; GFX11-CU:       ; %bb.0: ; %entry
6619; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6620; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6621; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
6622; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
6623; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
6624; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6625; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6626; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6627; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6628; GFX11-CU-NEXT:    s_endpgm
6629    i32* %in, i32* %out) {
6630entry:
6631  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") unordered, align 4
6632  store i32 %val, i32* %out
6633  ret void
6634}
6635
6636define amdgpu_kernel void @flat_workgroup_one_as_monotonic_load(
6637; GFX7-LABEL: flat_workgroup_one_as_monotonic_load:
6638; GFX7:       ; %bb.0: ; %entry
6639; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6640; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6641; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6642; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6643; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6644; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6645; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6646; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6647; GFX7-NEXT:    flat_store_dword v[0:1], v2
6648; GFX7-NEXT:    s_endpgm
6649;
6650; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_load:
6651; GFX10-WGP:       ; %bb.0: ; %entry
6652; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6653; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6654; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6655; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6656; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
6657; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6658; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6659; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6660; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6661; GFX10-WGP-NEXT:    s_endpgm
6662;
6663; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_load:
6664; GFX10-CU:       ; %bb.0: ; %entry
6665; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6666; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6667; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6668; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6669; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6670; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6671; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6672; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6673; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6674; GFX10-CU-NEXT:    s_endpgm
6675;
6676; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_load:
6677; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6678; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6679; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6680; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6681; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6682; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
6683; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6684; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6685; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6686; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6687; SKIP-CACHE-INV-NEXT:    s_endpgm
6688;
6689; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load:
6690; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6691; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6692; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6693; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6694; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6695; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6696; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6697; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6698; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6699; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6700; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6701;
6702; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load:
6703; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6704; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6705; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6706; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6707; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6708; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
6709; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6710; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6711; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6712; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6713; GFX90A-TGSPLIT-NEXT:    s_endpgm
6714;
6715; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load:
6716; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6717; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6718; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6719; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6720; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6721; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
6722; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6723; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6724; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6725; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6726; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6727;
6728; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load:
6729; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6730; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6731; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6732; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6733; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6734; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
6735; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6736; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6737; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6738; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6739; GFX940-TGSPLIT-NEXT:    s_endpgm
6740;
6741; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_load:
6742; GFX11-WGP:       ; %bb.0: ; %entry
6743; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6744; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6745; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
6746; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
6747; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
6748; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6749; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6750; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6751; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6752; GFX11-WGP-NEXT:    s_endpgm
6753;
6754; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_load:
6755; GFX11-CU:       ; %bb.0: ; %entry
6756; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6757; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6758; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
6759; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
6760; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
6761; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6762; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6763; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6764; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6765; GFX11-CU-NEXT:    s_endpgm
6766    i32* %in, i32* %out) {
6767entry:
6768  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") monotonic, align 4
6769  store i32 %val, i32* %out
6770  ret void
6771}
6772
6773define amdgpu_kernel void @flat_workgroup_one_as_acquire_load(
6774; GFX7-LABEL: flat_workgroup_one_as_acquire_load:
6775; GFX7:       ; %bb.0: ; %entry
6776; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6777; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6778; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6779; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6780; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6781; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6782; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6783; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6784; GFX7-NEXT:    flat_store_dword v[0:1], v2
6785; GFX7-NEXT:    s_endpgm
6786;
6787; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_load:
6788; GFX10-WGP:       ; %bb.0: ; %entry
6789; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6790; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6791; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6792; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6793; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
6794; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6795; GFX10-WGP-NEXT:    buffer_gl0_inv
6796; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6797; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6798; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6799; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6800; GFX10-WGP-NEXT:    s_endpgm
6801;
6802; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_load:
6803; GFX10-CU:       ; %bb.0: ; %entry
6804; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6805; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6806; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6807; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6808; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6809; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6810; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6811; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6812; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6813; GFX10-CU-NEXT:    s_endpgm
6814;
6815; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_load:
6816; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6817; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6818; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6819; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6820; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6821; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
6822; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6823; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6824; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6825; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6826; SKIP-CACHE-INV-NEXT:    s_endpgm
6827;
6828; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_load:
6829; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6830; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6831; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6832; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6833; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6834; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6835; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6836; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6837; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6838; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6839; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6840;
6841; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_load:
6842; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6843; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6844; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6845; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6846; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6847; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
6848; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6849; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6850; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6851; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6852; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6853; GFX90A-TGSPLIT-NEXT:    s_endpgm
6854;
6855; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_load:
6856; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6857; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6858; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6859; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6860; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6861; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
6862; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6863; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6864; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6865; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6866; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6867;
6868; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_load:
6869; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6870; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6871; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6872; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6873; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6874; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
6875; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6876; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
6877; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6878; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6879; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6880; GFX940-TGSPLIT-NEXT:    s_endpgm
6881;
6882; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_load:
6883; GFX11-WGP:       ; %bb.0: ; %entry
6884; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6885; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6886; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
6887; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
6888; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
6889; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
6890; GFX11-WGP-NEXT:    buffer_gl0_inv
6891; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6892; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6893; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6894; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6895; GFX11-WGP-NEXT:    s_endpgm
6896;
6897; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_load:
6898; GFX11-CU:       ; %bb.0: ; %entry
6899; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6900; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6901; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
6902; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
6903; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
6904; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6905; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6906; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6907; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6908; GFX11-CU-NEXT:    s_endpgm
6909    i32* %in, i32* %out) {
6910entry:
6911  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") acquire, align 4
6912  store i32 %val, i32* %out
6913  ret void
6914}
6915
6916define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_load(
6917; GFX7-LABEL: flat_workgroup_one_as_seq_cst_load:
6918; GFX7:       ; %bb.0: ; %entry
6919; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6920; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6921; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6922; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6923; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6924; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6925; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6926; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6927; GFX7-NEXT:    flat_store_dword v[0:1], v2
6928; GFX7-NEXT:    s_endpgm
6929;
6930; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_load:
6931; GFX10-WGP:       ; %bb.0: ; %entry
6932; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6933; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6934; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6935; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6936; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6937; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6938; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
6939; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6940; GFX10-WGP-NEXT:    buffer_gl0_inv
6941; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6942; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6943; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6944; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6945; GFX10-WGP-NEXT:    s_endpgm
6946;
6947; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_load:
6948; GFX10-CU:       ; %bb.0: ; %entry
6949; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6950; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6951; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6952; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6953; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6954; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6955; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6956; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6957; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6958; GFX10-CU-NEXT:    s_endpgm
6959;
6960; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_load:
6961; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6962; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6963; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6964; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6965; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6966; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
6967; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6968; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6969; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6970; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6971; SKIP-CACHE-INV-NEXT:    s_endpgm
6972;
6973; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load:
6974; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6975; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6976; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6977; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6978; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6979; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6980; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6981; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6982; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6983; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6984; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6985;
6986; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load:
6987; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6988; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6989; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6990; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6991; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6992; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6993; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
6994; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6995; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6996; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6997; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6998; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6999; GFX90A-TGSPLIT-NEXT:    s_endpgm
7000;
7001; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load:
7002; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7003; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7004; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7005; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7006; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7007; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
7008; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7009; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7010; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7011; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7012; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7013;
7014; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load:
7015; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7016; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7017; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7018; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7019; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7020; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7021; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0
7022; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7023; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
7024; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7025; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7026; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7027; GFX940-TGSPLIT-NEXT:    s_endpgm
7028;
7029; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_load:
7030; GFX11-WGP:       ; %bb.0: ; %entry
7031; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7032; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7033; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
7034; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
7035; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
7036; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7037; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
7038; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
7039; GFX11-WGP-NEXT:    buffer_gl0_inv
7040; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7041; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7042; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7043; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7044; GFX11-WGP-NEXT:    s_endpgm
7045;
7046; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_load:
7047; GFX11-CU:       ; %bb.0: ; %entry
7048; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7049; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7050; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
7051; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
7052; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
7053; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7054; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7055; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7056; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7057; GFX11-CU-NEXT:    s_endpgm
7058    i32* %in, i32* %out) {
7059entry:
7060  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") seq_cst, align 4
7061  store i32 %val, i32* %out
7062  ret void
7063}
7064
7065define amdgpu_kernel void @flat_workgroup_one_as_unordered_store(
7066; GFX7-LABEL: flat_workgroup_one_as_unordered_store:
7067; GFX7:       ; %bb.0: ; %entry
7068; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7069; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7070; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7071; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7072; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7073; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7074; GFX7-NEXT:    flat_store_dword v[0:1], v2
7075; GFX7-NEXT:    s_endpgm
7076;
7077; GFX10-WGP-LABEL: flat_workgroup_one_as_unordered_store:
7078; GFX10-WGP:       ; %bb.0: ; %entry
7079; GFX10-WGP-NEXT:    s_clause 0x1
7080; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7081; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7082; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7083; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7084; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7085; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7086; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7087; GFX10-WGP-NEXT:    s_endpgm
7088;
7089; GFX10-CU-LABEL: flat_workgroup_one_as_unordered_store:
7090; GFX10-CU:       ; %bb.0: ; %entry
7091; GFX10-CU-NEXT:    s_clause 0x1
7092; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7093; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7094; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7095; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7096; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7097; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7098; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7099; GFX10-CU-NEXT:    s_endpgm
7100;
7101; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_unordered_store:
7102; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7103; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
7104; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
7105; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7106; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7107; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7108; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7109; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7110; SKIP-CACHE-INV-NEXT:    s_endpgm
7111;
7112; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_store:
7113; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7114; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7115; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7116; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7117; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7118; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7119; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7120; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7121;
7122; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_store:
7123; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7124; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7125; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7126; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7127; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7128; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7129; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7130; GFX90A-TGSPLIT-NEXT:    s_endpgm
7131;
7132; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_store:
7133; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7134; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7135; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7136; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7137; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7138; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7139; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7140; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7141;
7142; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_store:
7143; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7144; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7145; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7146; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7147; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7148; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7149; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7150; GFX940-TGSPLIT-NEXT:    s_endpgm
7151;
7152; GFX11-WGP-LABEL: flat_workgroup_one_as_unordered_store:
7153; GFX11-WGP:       ; %bb.0: ; %entry
7154; GFX11-WGP-NEXT:    s_clause 0x1
7155; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7156; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
7157; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7158; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7159; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7160; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7161; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7162; GFX11-WGP-NEXT:    s_endpgm
7163;
7164; GFX11-CU-LABEL: flat_workgroup_one_as_unordered_store:
7165; GFX11-CU:       ; %bb.0: ; %entry
7166; GFX11-CU-NEXT:    s_clause 0x1
7167; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7168; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
7169; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7170; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7171; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7172; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7173; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7174; GFX11-CU-NEXT:    s_endpgm
7175    i32 %in, i32* %out) {
7176entry:
7177  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") unordered, align 4
7178  ret void
7179}
7180
7181define amdgpu_kernel void @flat_workgroup_one_as_monotonic_store(
7182; GFX7-LABEL: flat_workgroup_one_as_monotonic_store:
7183; GFX7:       ; %bb.0: ; %entry
7184; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7185; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7186; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7187; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7188; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7189; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7190; GFX7-NEXT:    flat_store_dword v[0:1], v2
7191; GFX7-NEXT:    s_endpgm
7192;
7193; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_store:
7194; GFX10-WGP:       ; %bb.0: ; %entry
7195; GFX10-WGP-NEXT:    s_clause 0x1
7196; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7197; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7198; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7199; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7200; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7201; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7202; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7203; GFX10-WGP-NEXT:    s_endpgm
7204;
7205; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_store:
7206; GFX10-CU:       ; %bb.0: ; %entry
7207; GFX10-CU-NEXT:    s_clause 0x1
7208; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7209; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7210; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7211; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7212; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7213; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7214; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7215; GFX10-CU-NEXT:    s_endpgm
7216;
7217; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_store:
7218; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7219; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
7220; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
7221; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7222; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7223; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7224; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7225; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7226; SKIP-CACHE-INV-NEXT:    s_endpgm
7227;
7228; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store:
7229; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7230; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7231; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7232; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7233; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7234; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7235; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7236; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7237;
7238; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store:
7239; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7240; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7241; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7242; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7243; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7244; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7245; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7246; GFX90A-TGSPLIT-NEXT:    s_endpgm
7247;
7248; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store:
7249; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7250; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7251; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7252; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7253; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7254; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7255; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
7256; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7257;
7258; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store:
7259; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7260; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7261; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7262; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7263; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7264; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7265; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
7266; GFX940-TGSPLIT-NEXT:    s_endpgm
7267;
7268; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_store:
7269; GFX11-WGP:       ; %bb.0: ; %entry
7270; GFX11-WGP-NEXT:    s_clause 0x1
7271; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7272; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
7273; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7274; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7275; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7276; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7277; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7278; GFX11-WGP-NEXT:    s_endpgm
7279;
7280; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_store:
7281; GFX11-CU:       ; %bb.0: ; %entry
7282; GFX11-CU-NEXT:    s_clause 0x1
7283; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7284; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
7285; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7286; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7287; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7288; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7289; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7290; GFX11-CU-NEXT:    s_endpgm
7291    i32 %in, i32* %out) {
7292entry:
7293  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") monotonic, align 4
7294  ret void
7295}
7296
7297define amdgpu_kernel void @flat_workgroup_one_as_release_store(
7298; GFX7-LABEL: flat_workgroup_one_as_release_store:
7299; GFX7:       ; %bb.0: ; %entry
7300; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7301; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7302; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7303; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7304; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7305; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7306; GFX7-NEXT:    flat_store_dword v[0:1], v2
7307; GFX7-NEXT:    s_endpgm
7308;
7309; GFX10-WGP-LABEL: flat_workgroup_one_as_release_store:
7310; GFX10-WGP:       ; %bb.0: ; %entry
7311; GFX10-WGP-NEXT:    s_clause 0x1
7312; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7313; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7314; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7315; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7316; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7317; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7318; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7319; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7320; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7321; GFX10-WGP-NEXT:    s_endpgm
7322;
7323; GFX10-CU-LABEL: flat_workgroup_one_as_release_store:
7324; GFX10-CU:       ; %bb.0: ; %entry
7325; GFX10-CU-NEXT:    s_clause 0x1
7326; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7327; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7328; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7329; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7330; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7331; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7332; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7333; GFX10-CU-NEXT:    s_endpgm
7334;
7335; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_store:
7336; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7337; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
7338; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
7339; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7340; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7341; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7342; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7343; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7344; SKIP-CACHE-INV-NEXT:    s_endpgm
7345;
7346; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_store:
7347; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7348; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7349; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7350; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7351; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7352; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7353; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7354; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7355;
7356; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_store:
7357; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7358; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7359; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7360; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7361; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7362; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7363; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7364; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7365; GFX90A-TGSPLIT-NEXT:    s_endpgm
7366;
7367; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_store:
7368; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7369; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7370; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7371; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7372; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7373; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7374; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
7375; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7376;
7377; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_store:
7378; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7379; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7380; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7381; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7382; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7383; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7384; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7385; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
7386; GFX940-TGSPLIT-NEXT:    s_endpgm
7387;
7388; GFX11-WGP-LABEL: flat_workgroup_one_as_release_store:
7389; GFX11-WGP:       ; %bb.0: ; %entry
7390; GFX11-WGP-NEXT:    s_clause 0x1
7391; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7392; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
7393; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7394; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7395; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7396; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7397; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
7398; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7399; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7400; GFX11-WGP-NEXT:    s_endpgm
7401;
7402; GFX11-CU-LABEL: flat_workgroup_one_as_release_store:
7403; GFX11-CU:       ; %bb.0: ; %entry
7404; GFX11-CU-NEXT:    s_clause 0x1
7405; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7406; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
7407; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7408; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7409; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7410; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7411; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7412; GFX11-CU-NEXT:    s_endpgm
7413    i32 %in, i32* %out) {
7414entry:
7415  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") release, align 4
7416  ret void
7417}
7418
7419define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_store(
7420; GFX7-LABEL: flat_workgroup_one_as_seq_cst_store:
7421; GFX7:       ; %bb.0: ; %entry
7422; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7423; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7424; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7425; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7426; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7427; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7428; GFX7-NEXT:    flat_store_dword v[0:1], v2
7429; GFX7-NEXT:    s_endpgm
7430;
7431; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_store:
7432; GFX10-WGP:       ; %bb.0: ; %entry
7433; GFX10-WGP-NEXT:    s_clause 0x1
7434; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7435; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7436; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7437; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7438; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7439; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7440; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7441; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7442; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7443; GFX10-WGP-NEXT:    s_endpgm
7444;
7445; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_store:
7446; GFX10-CU:       ; %bb.0: ; %entry
7447; GFX10-CU-NEXT:    s_clause 0x1
7448; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7449; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7450; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7451; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7452; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7453; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7454; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7455; GFX10-CU-NEXT:    s_endpgm
7456;
7457; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_store:
7458; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7459; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
7460; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
7461; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7462; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7463; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7464; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7465; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7466; SKIP-CACHE-INV-NEXT:    s_endpgm
7467;
7468; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store:
7469; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7470; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7471; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7472; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7473; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7474; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7475; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7476; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7477;
7478; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store:
7479; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7480; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7481; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7482; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7483; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7484; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7485; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7486; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7487; GFX90A-TGSPLIT-NEXT:    s_endpgm
7488;
7489; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store:
7490; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7491; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7492; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7493; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7494; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7495; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7496; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
7497; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7498;
7499; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store:
7500; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7501; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7502; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7503; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7504; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7505; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7506; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7507; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0
7508; GFX940-TGSPLIT-NEXT:    s_endpgm
7509;
7510; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_store:
7511; GFX11-WGP:       ; %bb.0: ; %entry
7512; GFX11-WGP-NEXT:    s_clause 0x1
7513; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7514; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
7515; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7516; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7517; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7518; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7519; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
7520; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7521; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7522; GFX11-WGP-NEXT:    s_endpgm
7523;
7524; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_store:
7525; GFX11-CU:       ; %bb.0: ; %entry
7526; GFX11-CU-NEXT:    s_clause 0x1
7527; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7528; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
7529; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7530; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7531; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7532; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7533; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7534; GFX11-CU-NEXT:    s_endpgm
7535    i32 %in, i32* %out) {
7536entry:
7537  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") seq_cst, align 4
7538  ret void
7539}
7540
7541define amdgpu_kernel void @flat_workgroup_one_as_monotonic_atomicrmw(
7542; GFX7-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
7543; GFX7:       ; %bb.0: ; %entry
7544; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7545; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7546; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7547; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7548; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7549; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7550; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7551; GFX7-NEXT:    s_endpgm
7552;
7553; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
7554; GFX10-WGP:       ; %bb.0: ; %entry
7555; GFX10-WGP-NEXT:    s_clause 0x1
7556; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7557; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7558; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7559; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7560; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7561; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7562; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7563; GFX10-WGP-NEXT:    s_endpgm
7564;
7565; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
7566; GFX10-CU:       ; %bb.0: ; %entry
7567; GFX10-CU-NEXT:    s_clause 0x1
7568; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7569; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7570; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7571; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7572; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7573; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7574; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7575; GFX10-CU-NEXT:    s_endpgm
7576;
7577; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
7578; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7579; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7580; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7581; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7582; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7583; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7584; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7585; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7586; SKIP-CACHE-INV-NEXT:    s_endpgm
7587;
7588; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
7589; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7590; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7591; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7592; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7593; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7594; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7595; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7596; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7597;
7598; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
7599; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7600; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7601; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7602; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7603; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7604; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7605; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7606; GFX90A-TGSPLIT-NEXT:    s_endpgm
7607;
7608; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
7609; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7610; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7611; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7612; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7613; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7614; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7615; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7616; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7617;
7618; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
7619; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7620; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7621; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7622; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7623; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7624; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7625; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7626; GFX940-TGSPLIT-NEXT:    s_endpgm
7627;
7628; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
7629; GFX11-WGP:       ; %bb.0: ; %entry
7630; GFX11-WGP-NEXT:    s_clause 0x1
7631; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7632; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
7633; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7634; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7635; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7636; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7637; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7638; GFX11-WGP-NEXT:    s_endpgm
7639;
7640; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
7641; GFX11-CU:       ; %bb.0: ; %entry
7642; GFX11-CU-NEXT:    s_clause 0x1
7643; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7644; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7645; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7646; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7647; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7648; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7649; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7650; GFX11-CU-NEXT:    s_endpgm
7651    i32* %out, i32 %in) {
7652entry:
7653  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") monotonic
7654  ret void
7655}
7656
7657define amdgpu_kernel void @flat_workgroup_one_as_acquire_atomicrmw(
7658; GFX7-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
7659; GFX7:       ; %bb.0: ; %entry
7660; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7661; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7662; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7663; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7664; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7665; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7666; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7667; GFX7-NEXT:    s_endpgm
7668;
7669; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
7670; GFX10-WGP:       ; %bb.0: ; %entry
7671; GFX10-WGP-NEXT:    s_clause 0x1
7672; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7673; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7674; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7675; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7676; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7677; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7678; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7679; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7680; GFX10-WGP-NEXT:    buffer_gl0_inv
7681; GFX10-WGP-NEXT:    s_endpgm
7682;
7683; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
7684; GFX10-CU:       ; %bb.0: ; %entry
7685; GFX10-CU-NEXT:    s_clause 0x1
7686; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7687; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7688; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7689; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7690; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7691; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7692; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7693; GFX10-CU-NEXT:    s_endpgm
7694;
7695; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
7696; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7697; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7698; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7699; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7700; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7701; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7702; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7703; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7704; SKIP-CACHE-INV-NEXT:    s_endpgm
7705;
7706; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
7707; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7708; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7709; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7710; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7711; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7712; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7713; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7714; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7715;
7716; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
7717; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7718; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7719; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7720; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7721; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7722; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7723; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7724; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7725; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7726; GFX90A-TGSPLIT-NEXT:    s_endpgm
7727;
7728; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
7729; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7730; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7731; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7732; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7733; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7734; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7735; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7736; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7737;
7738; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
7739; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7740; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7741; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7742; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7743; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7744; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7745; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7746; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7747; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
7748; GFX940-TGSPLIT-NEXT:    s_endpgm
7749;
7750; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
7751; GFX11-WGP:       ; %bb.0: ; %entry
7752; GFX11-WGP-NEXT:    s_clause 0x1
7753; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7754; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
7755; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7756; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7757; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7758; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7759; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7760; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7761; GFX11-WGP-NEXT:    buffer_gl0_inv
7762; GFX11-WGP-NEXT:    s_endpgm
7763;
7764; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
7765; GFX11-CU:       ; %bb.0: ; %entry
7766; GFX11-CU-NEXT:    s_clause 0x1
7767; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7768; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7769; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7770; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7771; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7772; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7773; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7774; GFX11-CU-NEXT:    s_endpgm
7775    i32* %out, i32 %in) {
7776entry:
7777  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acquire
7778  ret void
7779}
7780
7781define amdgpu_kernel void @flat_workgroup_one_as_release_atomicrmw(
7782; GFX7-LABEL: flat_workgroup_one_as_release_atomicrmw:
7783; GFX7:       ; %bb.0: ; %entry
7784; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7785; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7786; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7787; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7788; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7789; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7790; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7791; GFX7-NEXT:    s_endpgm
7792;
7793; GFX10-WGP-LABEL: flat_workgroup_one_as_release_atomicrmw:
7794; GFX10-WGP:       ; %bb.0: ; %entry
7795; GFX10-WGP-NEXT:    s_clause 0x1
7796; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7797; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7798; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7799; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7800; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7801; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7802; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7803; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7804; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7805; GFX10-WGP-NEXT:    s_endpgm
7806;
7807; GFX10-CU-LABEL: flat_workgroup_one_as_release_atomicrmw:
7808; GFX10-CU:       ; %bb.0: ; %entry
7809; GFX10-CU-NEXT:    s_clause 0x1
7810; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7811; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7812; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7813; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7814; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7815; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7816; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7817; GFX10-CU-NEXT:    s_endpgm
7818;
7819; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_atomicrmw:
7820; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7821; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7822; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7823; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7824; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7825; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7826; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7827; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7828; SKIP-CACHE-INV-NEXT:    s_endpgm
7829;
7830; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw:
7831; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7832; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7833; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7834; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7835; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7836; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7837; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7838; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7839;
7840; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw:
7841; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7842; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7843; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7844; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7845; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7846; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7847; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7848; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7849; GFX90A-TGSPLIT-NEXT:    s_endpgm
7850;
7851; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw:
7852; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7853; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7854; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7855; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7856; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7857; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7858; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7859; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7860;
7861; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw:
7862; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7863; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7864; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7865; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7866; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7867; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7868; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7869; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7870; GFX940-TGSPLIT-NEXT:    s_endpgm
7871;
7872; GFX11-WGP-LABEL: flat_workgroup_one_as_release_atomicrmw:
7873; GFX11-WGP:       ; %bb.0: ; %entry
7874; GFX11-WGP-NEXT:    s_clause 0x1
7875; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7876; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
7877; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7878; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7879; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7880; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7881; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
7882; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7883; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7884; GFX11-WGP-NEXT:    s_endpgm
7885;
7886; GFX11-CU-LABEL: flat_workgroup_one_as_release_atomicrmw:
7887; GFX11-CU:       ; %bb.0: ; %entry
7888; GFX11-CU-NEXT:    s_clause 0x1
7889; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7890; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7891; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7892; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7893; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7894; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7895; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7896; GFX11-CU-NEXT:    s_endpgm
7897    i32* %out, i32 %in) {
7898entry:
7899  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") release
7900  ret void
7901}
7902
7903define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_atomicrmw(
7904; GFX7-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
7905; GFX7:       ; %bb.0: ; %entry
7906; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7907; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7908; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7909; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7910; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7911; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7912; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7913; GFX7-NEXT:    s_endpgm
7914;
7915; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
7916; GFX10-WGP:       ; %bb.0: ; %entry
7917; GFX10-WGP-NEXT:    s_clause 0x1
7918; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7919; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7920; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7921; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7922; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7923; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7924; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7925; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7926; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7927; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7928; GFX10-WGP-NEXT:    buffer_gl0_inv
7929; GFX10-WGP-NEXT:    s_endpgm
7930;
7931; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
7932; GFX10-CU:       ; %bb.0: ; %entry
7933; GFX10-CU-NEXT:    s_clause 0x1
7934; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7935; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7936; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7937; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7938; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7939; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7940; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7941; GFX10-CU-NEXT:    s_endpgm
7942;
7943; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
7944; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7945; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7946; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7947; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7948; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7949; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7950; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7951; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7952; SKIP-CACHE-INV-NEXT:    s_endpgm
7953;
7954; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
7955; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7956; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7957; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7958; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7959; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7960; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7961; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7962; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7963;
7964; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
7965; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7966; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7967; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7968; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7969; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7970; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7971; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7972; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7973; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7974; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7975; GFX90A-TGSPLIT-NEXT:    s_endpgm
7976;
7977; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
7978; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7979; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7980; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7981; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7982; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7983; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7984; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7985; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7986;
7987; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
7988; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7989; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7990; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7991; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7992; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7993; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7994; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7995; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7996; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7997; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
7998; GFX940-TGSPLIT-NEXT:    s_endpgm
7999;
8000; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
8001; GFX11-WGP:       ; %bb.0: ; %entry
8002; GFX11-WGP-NEXT:    s_clause 0x1
8003; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8004; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8005; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8006; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8007; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8008; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8009; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8010; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8011; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8012; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8013; GFX11-WGP-NEXT:    buffer_gl0_inv
8014; GFX11-WGP-NEXT:    s_endpgm
8015;
8016; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
8017; GFX11-CU:       ; %bb.0: ; %entry
8018; GFX11-CU-NEXT:    s_clause 0x1
8019; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8020; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8021; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8022; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8023; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8024; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8025; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8026; GFX11-CU-NEXT:    s_endpgm
8027    i32* %out, i32 %in) {
8028entry:
8029  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acq_rel
8030  ret void
8031}
8032
8033define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_atomicrmw(
8034; GFX7-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
8035; GFX7:       ; %bb.0: ; %entry
8036; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8037; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8038; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8039; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8040; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8041; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8042; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
8043; GFX7-NEXT:    s_endpgm
8044;
8045; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
8046; GFX10-WGP:       ; %bb.0: ; %entry
8047; GFX10-WGP-NEXT:    s_clause 0x1
8048; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8049; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8050; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8051; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8052; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8053; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8054; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8055; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8056; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
8057; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8058; GFX10-WGP-NEXT:    buffer_gl0_inv
8059; GFX10-WGP-NEXT:    s_endpgm
8060;
8061; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
8062; GFX10-CU:       ; %bb.0: ; %entry
8063; GFX10-CU-NEXT:    s_clause 0x1
8064; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8065; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8066; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8067; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8068; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8069; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8070; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
8071; GFX10-CU-NEXT:    s_endpgm
8072;
8073; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
8074; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8075; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8076; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8077; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8078; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8079; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8080; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8081; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
8082; SKIP-CACHE-INV-NEXT:    s_endpgm
8083;
8084; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
8085; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8086; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8087; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8088; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8089; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8090; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8091; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8092; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8093;
8094; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
8095; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8096; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8097; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8098; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8099; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8100; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8101; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8102; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8103; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8104; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8105; GFX90A-TGSPLIT-NEXT:    s_endpgm
8106;
8107; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
8108; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8109; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8110; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8111; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8112; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8113; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8114; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8115; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8116;
8117; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
8118; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8119; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8120; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8121; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8122; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8123; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8124; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8125; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8126; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8127; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
8128; GFX940-TGSPLIT-NEXT:    s_endpgm
8129;
8130; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
8131; GFX11-WGP:       ; %bb.0: ; %entry
8132; GFX11-WGP-NEXT:    s_clause 0x1
8133; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8134; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8135; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8136; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8137; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8138; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8139; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8140; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8141; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8142; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8143; GFX11-WGP-NEXT:    buffer_gl0_inv
8144; GFX11-WGP-NEXT:    s_endpgm
8145;
8146; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
8147; GFX11-CU:       ; %bb.0: ; %entry
8148; GFX11-CU-NEXT:    s_clause 0x1
8149; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8150; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8151; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8152; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8153; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8154; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8155; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8156; GFX11-CU-NEXT:    s_endpgm
8157    i32* %out, i32 %in) {
8158entry:
8159  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") seq_cst
8160  ret void
8161}
8162
8163define amdgpu_kernel void @flat_workgroup_one_as_acquire_ret_atomicrmw(
8164; GFX7-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
8165; GFX7:       ; %bb.0: ; %entry
8166; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8167; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8168; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8169; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8170; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8171; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8172; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8173; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8174; GFX7-NEXT:    flat_store_dword v[0:1], v2
8175; GFX7-NEXT:    s_endpgm
8176;
8177; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
8178; GFX10-WGP:       ; %bb.0: ; %entry
8179; GFX10-WGP-NEXT:    s_clause 0x1
8180; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8181; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8182; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8183; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8184; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8185; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8186; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8187; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8188; GFX10-WGP-NEXT:    buffer_gl0_inv
8189; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8190; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8191; GFX10-WGP-NEXT:    s_endpgm
8192;
8193; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
8194; GFX10-CU:       ; %bb.0: ; %entry
8195; GFX10-CU-NEXT:    s_clause 0x1
8196; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8197; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8198; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8199; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8200; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8201; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8202; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8203; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8204; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8205; GFX10-CU-NEXT:    s_endpgm
8206;
8207; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
8208; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8209; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8210; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8211; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8212; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8213; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8214; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8215; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8216; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8217; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8218; SKIP-CACHE-INV-NEXT:    s_endpgm
8219;
8220; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
8221; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8222; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8223; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8224; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8225; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8226; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8227; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8228; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8229; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8230; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8231;
8232; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
8233; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8234; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8235; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8236; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8237; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8238; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8239; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8240; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8241; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8242; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8243; GFX90A-TGSPLIT-NEXT:    s_endpgm
8244;
8245; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
8246; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8247; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8248; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8249; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8250; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8251; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8252; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8253; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8254; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8255; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8256;
8257; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
8258; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8259; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8260; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8261; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8262; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8263; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8264; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8265; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8266; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
8267; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8268; GFX940-TGSPLIT-NEXT:    s_endpgm
8269;
8270; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
8271; GFX11-WGP:       ; %bb.0: ; %entry
8272; GFX11-WGP-NEXT:    s_clause 0x1
8273; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8274; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8275; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8276; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8277; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8278; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8279; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8280; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8281; GFX11-WGP-NEXT:    buffer_gl0_inv
8282; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8283; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8284; GFX11-WGP-NEXT:    s_endpgm
8285;
8286; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
8287; GFX11-CU:       ; %bb.0: ; %entry
8288; GFX11-CU-NEXT:    s_clause 0x1
8289; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8290; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8291; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8292; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8293; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8294; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8295; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8296; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8297; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8298; GFX11-CU-NEXT:    s_endpgm
8299    i32* %out, i32 %in) {
8300entry:
8301  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acquire
8302  store i32 %val, i32* %out, align 4
8303  ret void
8304}
8305
8306define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_ret_atomicrmw(
8307; GFX7-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
8308; GFX7:       ; %bb.0: ; %entry
8309; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8310; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8311; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8312; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8313; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8314; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8315; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8316; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8317; GFX7-NEXT:    flat_store_dword v[0:1], v2
8318; GFX7-NEXT:    s_endpgm
8319;
8320; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
8321; GFX10-WGP:       ; %bb.0: ; %entry
8322; GFX10-WGP-NEXT:    s_clause 0x1
8323; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8324; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8325; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8326; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8327; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8328; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8329; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8330; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8331; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8332; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8333; GFX10-WGP-NEXT:    buffer_gl0_inv
8334; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8335; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8336; GFX10-WGP-NEXT:    s_endpgm
8337;
8338; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
8339; GFX10-CU:       ; %bb.0: ; %entry
8340; GFX10-CU-NEXT:    s_clause 0x1
8341; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8342; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8343; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8344; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8345; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8346; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8347; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8348; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8349; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8350; GFX10-CU-NEXT:    s_endpgm
8351;
8352; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
8353; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8354; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8355; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8356; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8357; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8358; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8359; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8360; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8361; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8362; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8363; SKIP-CACHE-INV-NEXT:    s_endpgm
8364;
8365; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
8366; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8367; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8368; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8369; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8370; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8371; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8372; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8373; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8374; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8375; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8376;
8377; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
8378; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8379; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8380; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8381; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8382; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8383; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8384; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8385; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8386; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8387; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8388; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8389; GFX90A-TGSPLIT-NEXT:    s_endpgm
8390;
8391; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
8392; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8393; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8394; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8395; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8396; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8397; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8398; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8399; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8400; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8401; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8402;
8403; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
8404; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8405; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8406; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8407; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8408; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8409; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8410; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8411; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8412; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8413; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
8414; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8415; GFX940-TGSPLIT-NEXT:    s_endpgm
8416;
8417; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
8418; GFX11-WGP:       ; %bb.0: ; %entry
8419; GFX11-WGP-NEXT:    s_clause 0x1
8420; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8421; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8422; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8423; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8424; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8425; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8426; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8427; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8428; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8429; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8430; GFX11-WGP-NEXT:    buffer_gl0_inv
8431; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8432; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8433; GFX11-WGP-NEXT:    s_endpgm
8434;
8435; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
8436; GFX11-CU:       ; %bb.0: ; %entry
8437; GFX11-CU-NEXT:    s_clause 0x1
8438; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8439; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8440; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8441; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8442; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8443; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8444; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8445; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8446; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8447; GFX11-CU-NEXT:    s_endpgm
8448    i32* %out, i32 %in) {
8449entry:
8450  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acq_rel
8451  store i32 %val, i32* %out, align 4
8452  ret void
8453}
8454
8455define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_ret_atomicrmw(
8456; GFX7-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
8457; GFX7:       ; %bb.0: ; %entry
8458; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8459; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8460; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8461; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8462; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8463; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8464; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8465; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8466; GFX7-NEXT:    flat_store_dword v[0:1], v2
8467; GFX7-NEXT:    s_endpgm
8468;
8469; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
8470; GFX10-WGP:       ; %bb.0: ; %entry
8471; GFX10-WGP-NEXT:    s_clause 0x1
8472; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8473; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8474; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8475; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8476; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8477; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8478; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8479; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8480; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8481; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8482; GFX10-WGP-NEXT:    buffer_gl0_inv
8483; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8484; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8485; GFX10-WGP-NEXT:    s_endpgm
8486;
8487; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
8488; GFX10-CU:       ; %bb.0: ; %entry
8489; GFX10-CU-NEXT:    s_clause 0x1
8490; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8491; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8492; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8493; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8494; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8495; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8496; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8497; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8498; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8499; GFX10-CU-NEXT:    s_endpgm
8500;
8501; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
8502; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8503; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8504; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8505; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8506; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8507; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8508; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8509; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8510; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8511; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8512; SKIP-CACHE-INV-NEXT:    s_endpgm
8513;
8514; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
8515; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8516; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8517; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8518; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8519; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8520; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8521; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8522; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8523; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8524; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8525;
8526; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
8527; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8528; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8529; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8530; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8531; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8532; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8533; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8534; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8535; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8536; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8537; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8538; GFX90A-TGSPLIT-NEXT:    s_endpgm
8539;
8540; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
8541; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8542; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8543; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8544; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8545; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8546; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8547; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8548; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8549; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8550; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8551;
8552; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
8553; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8554; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8555; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8556; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8557; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8558; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8559; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8560; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8561; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8562; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
8563; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8564; GFX940-TGSPLIT-NEXT:    s_endpgm
8565;
8566; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
8567; GFX11-WGP:       ; %bb.0: ; %entry
8568; GFX11-WGP-NEXT:    s_clause 0x1
8569; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8570; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8571; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8572; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8573; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8574; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8575; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8576; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8577; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8578; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8579; GFX11-WGP-NEXT:    buffer_gl0_inv
8580; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8581; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8582; GFX11-WGP-NEXT:    s_endpgm
8583;
8584; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
8585; GFX11-CU:       ; %bb.0: ; %entry
8586; GFX11-CU-NEXT:    s_clause 0x1
8587; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8588; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8589; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8590; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8591; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8592; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8593; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8594; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8595; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8596; GFX11-CU-NEXT:    s_endpgm
8597    i32* %out, i32 %in) {
8598entry:
8599  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") seq_cst
8600  store i32 %val, i32* %out, align 4
8601  ret void
8602}
8603
8604define amdgpu_kernel void @flat_workgroup_one_as_monotonic_monotonic_cmpxchg(
8605; GFX7-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
8606; GFX7:       ; %bb.0: ; %entry
8607; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8608; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8609; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8610; GFX7-NEXT:    s_add_u32 s0, s0, 16
8611; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8612; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8613; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8614; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8615; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8616; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8617; GFX7-NEXT:    s_endpgm
8618;
8619; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
8620; GFX10-WGP:       ; %bb.0: ; %entry
8621; GFX10-WGP-NEXT:    s_clause 0x1
8622; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8623; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8624; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8625; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8626; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8627; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8628; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8629; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8630; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8631; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8632; GFX10-WGP-NEXT:    s_endpgm
8633;
8634; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
8635; GFX10-CU:       ; %bb.0: ; %entry
8636; GFX10-CU-NEXT:    s_clause 0x1
8637; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8638; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8639; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8640; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8641; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8642; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8643; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8644; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8645; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8646; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8647; GFX10-CU-NEXT:    s_endpgm
8648;
8649; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
8650; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8651; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8652; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8653; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8654; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8655; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8656; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8657; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8658; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8659; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8660; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8661; SKIP-CACHE-INV-NEXT:    s_endpgm
8662;
8663; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
8664; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8665; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8666; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8667; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8668; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8669; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8670; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8671; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8672;
8673; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
8674; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8675; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8676; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8677; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8678; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8679; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8680; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8681; GFX90A-TGSPLIT-NEXT:    s_endpgm
8682;
8683; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
8684; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8685; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8686; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8687; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8688; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8689; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8690; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8691; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8692;
8693; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
8694; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8695; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8696; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8697; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8698; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8699; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8700; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8701; GFX940-TGSPLIT-NEXT:    s_endpgm
8702;
8703; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
8704; GFX11-WGP:       ; %bb.0: ; %entry
8705; GFX11-WGP-NEXT:    s_clause 0x1
8706; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8707; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8708; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8709; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8710; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
8711; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8712; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8713; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8714; GFX11-WGP-NEXT:    s_endpgm
8715;
8716; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
8717; GFX11-CU:       ; %bb.0: ; %entry
8718; GFX11-CU-NEXT:    s_clause 0x1
8719; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8720; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8721; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8722; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8723; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
8724; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8725; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8726; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8727; GFX11-CU-NEXT:    s_endpgm
8728    i32* %out, i32 %in, i32 %old) {
8729entry:
8730  %gep = getelementptr i32, i32* %out, i32 4
8731  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic
8732  ret void
8733}
8734
8735define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_cmpxchg(
8736; GFX7-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
8737; GFX7:       ; %bb.0: ; %entry
8738; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8739; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8740; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8741; GFX7-NEXT:    s_add_u32 s0, s0, 16
8742; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8743; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8744; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8745; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8746; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8747; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8748; GFX7-NEXT:    s_endpgm
8749;
8750; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
8751; GFX10-WGP:       ; %bb.0: ; %entry
8752; GFX10-WGP-NEXT:    s_clause 0x1
8753; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8754; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8755; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8756; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8757; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8758; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8759; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8760; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8761; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8762; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8763; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8764; GFX10-WGP-NEXT:    buffer_gl0_inv
8765; GFX10-WGP-NEXT:    s_endpgm
8766;
8767; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
8768; GFX10-CU:       ; %bb.0: ; %entry
8769; GFX10-CU-NEXT:    s_clause 0x1
8770; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8771; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8772; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8773; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8774; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8775; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8776; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8777; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8778; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8779; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8780; GFX10-CU-NEXT:    s_endpgm
8781;
8782; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
8783; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8784; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8785; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8786; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8787; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8788; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8789; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8790; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8791; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8792; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8793; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8794; SKIP-CACHE-INV-NEXT:    s_endpgm
8795;
8796; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
8797; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8798; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8799; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8800; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8801; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8802; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8803; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8804; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8805;
8806; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
8807; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8808; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8809; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8810; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8811; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8812; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8813; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8814; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8815; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8816; GFX90A-TGSPLIT-NEXT:    s_endpgm
8817;
8818; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
8819; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8820; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8821; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8822; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8823; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8824; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8825; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8826; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8827;
8828; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
8829; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8830; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8831; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8832; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8833; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8834; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8835; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8836; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8837; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
8838; GFX940-TGSPLIT-NEXT:    s_endpgm
8839;
8840; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
8841; GFX11-WGP:       ; %bb.0: ; %entry
8842; GFX11-WGP-NEXT:    s_clause 0x1
8843; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8844; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8845; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8846; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8847; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
8848; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8849; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8850; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8851; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8852; GFX11-WGP-NEXT:    buffer_gl0_inv
8853; GFX11-WGP-NEXT:    s_endpgm
8854;
8855; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
8856; GFX11-CU:       ; %bb.0: ; %entry
8857; GFX11-CU-NEXT:    s_clause 0x1
8858; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8859; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8860; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8861; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8862; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
8863; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8864; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8865; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8866; GFX11-CU-NEXT:    s_endpgm
8867    i32* %out, i32 %in, i32 %old) {
8868entry:
8869  %gep = getelementptr i32, i32* %out, i32 4
8870  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
8871  ret void
8872}
8873
8874define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_cmpxchg(
8875; GFX7-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
8876; GFX7:       ; %bb.0: ; %entry
8877; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8878; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8879; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8880; GFX7-NEXT:    s_add_u32 s0, s0, 16
8881; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8882; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8883; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8884; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8885; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8886; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8887; GFX7-NEXT:    s_endpgm
8888;
8889; GFX10-WGP-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
8890; GFX10-WGP:       ; %bb.0: ; %entry
8891; GFX10-WGP-NEXT:    s_clause 0x1
8892; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8893; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8894; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8895; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8896; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8897; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8898; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8899; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8900; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8901; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8902; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8903; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8904; GFX10-WGP-NEXT:    s_endpgm
8905;
8906; GFX10-CU-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
8907; GFX10-CU:       ; %bb.0: ; %entry
8908; GFX10-CU-NEXT:    s_clause 0x1
8909; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8910; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8911; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8912; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8913; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8914; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8915; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8916; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8917; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8918; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8919; GFX10-CU-NEXT:    s_endpgm
8920;
8921; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
8922; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8923; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8924; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8925; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8926; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8927; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8928; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8929; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8930; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8931; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8932; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8933; SKIP-CACHE-INV-NEXT:    s_endpgm
8934;
8935; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
8936; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8937; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8938; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8939; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8940; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8941; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8942; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8943; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8944;
8945; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
8946; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8947; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8948; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8949; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8950; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8951; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8952; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8953; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8954; GFX90A-TGSPLIT-NEXT:    s_endpgm
8955;
8956; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
8957; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8958; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8959; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8960; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8961; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8962; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8963; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8964; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8965;
8966; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
8967; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8968; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8969; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8970; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8971; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8972; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8973; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8974; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8975; GFX940-TGSPLIT-NEXT:    s_endpgm
8976;
8977; GFX11-WGP-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
8978; GFX11-WGP:       ; %bb.0: ; %entry
8979; GFX11-WGP-NEXT:    s_clause 0x1
8980; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8981; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8982; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8983; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8984; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
8985; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8986; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8987; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8988; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8989; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8990; GFX11-WGP-NEXT:    s_endpgm
8991;
8992; GFX11-CU-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
8993; GFX11-CU:       ; %bb.0: ; %entry
8994; GFX11-CU-NEXT:    s_clause 0x1
8995; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8996; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8997; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8998; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8999; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9000; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9001; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9002; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9003; GFX11-CU-NEXT:    s_endpgm
9004    i32* %out, i32 %in, i32 %old) {
9005entry:
9006  %gep = getelementptr i32, i32* %out, i32 4
9007  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic
9008  ret void
9009}
9010
9011define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_cmpxchg(
9012; GFX7-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
9013; GFX7:       ; %bb.0: ; %entry
9014; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9015; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9016; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9017; GFX7-NEXT:    s_add_u32 s0, s0, 16
9018; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9019; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9020; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9021; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9022; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9023; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9024; GFX7-NEXT:    s_endpgm
9025;
9026; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
9027; GFX10-WGP:       ; %bb.0: ; %entry
9028; GFX10-WGP-NEXT:    s_clause 0x1
9029; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9030; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9031; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9032; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9033; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9034; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9035; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9036; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9037; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9038; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9039; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9040; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9041; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9042; GFX10-WGP-NEXT:    buffer_gl0_inv
9043; GFX10-WGP-NEXT:    s_endpgm
9044;
9045; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
9046; GFX10-CU:       ; %bb.0: ; %entry
9047; GFX10-CU-NEXT:    s_clause 0x1
9048; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9049; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9050; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9051; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9052; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9053; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9054; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9055; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9056; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9057; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9058; GFX10-CU-NEXT:    s_endpgm
9059;
9060; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
9061; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9062; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9063; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9064; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9065; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9066; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9067; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9068; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9069; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9070; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9071; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9072; SKIP-CACHE-INV-NEXT:    s_endpgm
9073;
9074; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
9075; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9076; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9077; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9078; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9079; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9080; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9081; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9082; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9083;
9084; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
9085; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9086; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9087; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9088; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9089; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9090; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9091; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9092; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9093; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9094; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9095; GFX90A-TGSPLIT-NEXT:    s_endpgm
9096;
9097; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
9098; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9099; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9100; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9101; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9102; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9103; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9104; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9105; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9106;
9107; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
9108; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9109; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9110; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9111; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9112; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9113; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9114; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9115; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9116; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9117; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
9118; GFX940-TGSPLIT-NEXT:    s_endpgm
9119;
9120; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
9121; GFX11-WGP:       ; %bb.0: ; %entry
9122; GFX11-WGP-NEXT:    s_clause 0x1
9123; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9124; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9125; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9126; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9127; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9128; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9129; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9130; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9131; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9132; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9133; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9134; GFX11-WGP-NEXT:    buffer_gl0_inv
9135; GFX11-WGP-NEXT:    s_endpgm
9136;
9137; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
9138; GFX11-CU:       ; %bb.0: ; %entry
9139; GFX11-CU-NEXT:    s_clause 0x1
9140; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9141; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9142; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9143; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9144; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9145; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9146; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9147; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9148; GFX11-CU-NEXT:    s_endpgm
9149    i32* %out, i32 %in, i32 %old) {
9150entry:
9151  %gep = getelementptr i32, i32* %out, i32 4
9152  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
9153  ret void
9154}
9155
9156define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_cmpxchg(
9157; GFX7-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
9158; GFX7:       ; %bb.0: ; %entry
9159; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9160; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9161; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9162; GFX7-NEXT:    s_add_u32 s0, s0, 16
9163; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9164; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9165; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9166; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9167; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9168; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9169; GFX7-NEXT:    s_endpgm
9170;
9171; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
9172; GFX10-WGP:       ; %bb.0: ; %entry
9173; GFX10-WGP-NEXT:    s_clause 0x1
9174; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9175; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9176; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9177; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9178; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9179; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9180; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9181; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9182; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9183; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9184; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9185; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9186; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9187; GFX10-WGP-NEXT:    buffer_gl0_inv
9188; GFX10-WGP-NEXT:    s_endpgm
9189;
9190; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
9191; GFX10-CU:       ; %bb.0: ; %entry
9192; GFX10-CU-NEXT:    s_clause 0x1
9193; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9194; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9195; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9196; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9197; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9198; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9199; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9200; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9201; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9202; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9203; GFX10-CU-NEXT:    s_endpgm
9204;
9205; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
9206; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9207; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9208; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9209; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9210; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9211; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9212; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9213; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9214; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9215; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9216; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9217; SKIP-CACHE-INV-NEXT:    s_endpgm
9218;
9219; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
9220; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9221; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9222; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9223; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9224; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9225; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9226; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9227; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9228;
9229; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
9230; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9231; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9232; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9233; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9234; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9235; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9236; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9237; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9238; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9239; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9240; GFX90A-TGSPLIT-NEXT:    s_endpgm
9241;
9242; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
9243; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9244; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9245; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9246; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9247; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9248; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9249; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9250; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9251;
9252; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
9253; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9254; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9255; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9256; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9257; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9258; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9259; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9260; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9261; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9262; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
9263; GFX940-TGSPLIT-NEXT:    s_endpgm
9264;
9265; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
9266; GFX11-WGP:       ; %bb.0: ; %entry
9267; GFX11-WGP-NEXT:    s_clause 0x1
9268; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9269; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9270; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9271; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9272; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9273; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9274; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9275; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9276; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9277; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9278; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9279; GFX11-WGP-NEXT:    buffer_gl0_inv
9280; GFX11-WGP-NEXT:    s_endpgm
9281;
9282; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
9283; GFX11-CU:       ; %bb.0: ; %entry
9284; GFX11-CU-NEXT:    s_clause 0x1
9285; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9286; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9287; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9288; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9289; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9290; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9291; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9292; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9293; GFX11-CU-NEXT:    s_endpgm
9294    i32* %out, i32 %in, i32 %old) {
9295entry:
9296  %gep = getelementptr i32, i32* %out, i32 4
9297  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
9298  ret void
9299}
9300
9301define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_cmpxchg(
9302; GFX7-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
9303; GFX7:       ; %bb.0: ; %entry
9304; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9305; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9306; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9307; GFX7-NEXT:    s_add_u32 s0, s0, 16
9308; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9309; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9310; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9311; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9312; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9313; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9314; GFX7-NEXT:    s_endpgm
9315;
9316; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
9317; GFX10-WGP:       ; %bb.0: ; %entry
9318; GFX10-WGP-NEXT:    s_clause 0x1
9319; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9320; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9321; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9322; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9323; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9324; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9325; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9326; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9327; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9328; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9329; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9330; GFX10-WGP-NEXT:    buffer_gl0_inv
9331; GFX10-WGP-NEXT:    s_endpgm
9332;
9333; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
9334; GFX10-CU:       ; %bb.0: ; %entry
9335; GFX10-CU-NEXT:    s_clause 0x1
9336; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9337; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9338; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9339; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9340; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9341; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9342; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9343; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9344; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9345; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9346; GFX10-CU-NEXT:    s_endpgm
9347;
9348; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
9349; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9350; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9351; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9352; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9353; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9354; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9355; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9356; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9357; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9358; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9359; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9360; SKIP-CACHE-INV-NEXT:    s_endpgm
9361;
9362; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
9363; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9364; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9365; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9366; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9367; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9368; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9369; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9370; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9371;
9372; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
9373; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9374; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9375; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9376; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9377; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9378; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9379; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9380; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9381; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9382; GFX90A-TGSPLIT-NEXT:    s_endpgm
9383;
9384; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
9385; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9386; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9387; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9388; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9389; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9390; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9391; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9392; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9393;
9394; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
9395; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9396; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9397; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9398; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9399; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9400; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9401; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9402; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9403; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
9404; GFX940-TGSPLIT-NEXT:    s_endpgm
9405;
9406; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
9407; GFX11-WGP:       ; %bb.0: ; %entry
9408; GFX11-WGP-NEXT:    s_clause 0x1
9409; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9410; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9411; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9412; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9413; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9414; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9415; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9416; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9417; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9418; GFX11-WGP-NEXT:    buffer_gl0_inv
9419; GFX11-WGP-NEXT:    s_endpgm
9420;
9421; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
9422; GFX11-CU:       ; %bb.0: ; %entry
9423; GFX11-CU-NEXT:    s_clause 0x1
9424; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9425; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9426; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9427; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9428; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9429; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9430; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9431; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9432; GFX11-CU-NEXT:    s_endpgm
9433    i32* %out, i32 %in, i32 %old) {
9434entry:
9435  %gep = getelementptr i32, i32* %out, i32 4
9436  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic acquire
9437  ret void
9438}
9439
9440define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_cmpxchg(
9441; GFX7-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
9442; GFX7:       ; %bb.0: ; %entry
9443; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9444; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9445; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9446; GFX7-NEXT:    s_add_u32 s0, s0, 16
9447; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9448; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9449; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9450; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9451; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9452; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9453; GFX7-NEXT:    s_endpgm
9454;
9455; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
9456; GFX10-WGP:       ; %bb.0: ; %entry
9457; GFX10-WGP-NEXT:    s_clause 0x1
9458; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9459; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9460; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9461; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9462; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9463; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9464; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9465; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9466; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9467; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9468; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9469; GFX10-WGP-NEXT:    buffer_gl0_inv
9470; GFX10-WGP-NEXT:    s_endpgm
9471;
9472; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
9473; GFX10-CU:       ; %bb.0: ; %entry
9474; GFX10-CU-NEXT:    s_clause 0x1
9475; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9476; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9477; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9478; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9479; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9480; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9481; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9482; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9483; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9484; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9485; GFX10-CU-NEXT:    s_endpgm
9486;
9487; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
9488; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9489; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9490; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9491; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9492; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9493; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9494; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9495; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9496; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9497; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9498; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9499; SKIP-CACHE-INV-NEXT:    s_endpgm
9500;
9501; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
9502; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9503; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9504; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9505; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9506; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9507; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9508; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9509; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9510;
9511; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
9512; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9513; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9514; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9515; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9516; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9517; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9518; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9519; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9520; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9521; GFX90A-TGSPLIT-NEXT:    s_endpgm
9522;
9523; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
9524; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9525; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9526; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9527; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9528; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9529; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9530; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9531; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9532;
9533; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
9534; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9535; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9536; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9537; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9538; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9539; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9540; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9541; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9542; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
9543; GFX940-TGSPLIT-NEXT:    s_endpgm
9544;
9545; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
9546; GFX11-WGP:       ; %bb.0: ; %entry
9547; GFX11-WGP-NEXT:    s_clause 0x1
9548; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9549; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9550; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9551; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9552; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9553; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9554; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9555; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9556; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9557; GFX11-WGP-NEXT:    buffer_gl0_inv
9558; GFX11-WGP-NEXT:    s_endpgm
9559;
9560; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
9561; GFX11-CU:       ; %bb.0: ; %entry
9562; GFX11-CU-NEXT:    s_clause 0x1
9563; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9564; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9565; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9566; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9567; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9568; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9569; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9570; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9571; GFX11-CU-NEXT:    s_endpgm
9572    i32* %out, i32 %in, i32 %old) {
9573entry:
9574  %gep = getelementptr i32, i32* %out, i32 4
9575  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
9576  ret void
9577}
9578
9579define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_cmpxchg(
9580; GFX7-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
9581; GFX7:       ; %bb.0: ; %entry
9582; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9583; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9584; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9585; GFX7-NEXT:    s_add_u32 s0, s0, 16
9586; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9587; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9588; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9589; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9590; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9591; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9592; GFX7-NEXT:    s_endpgm
9593;
9594; GFX10-WGP-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
9595; GFX10-WGP:       ; %bb.0: ; %entry
9596; GFX10-WGP-NEXT:    s_clause 0x1
9597; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9598; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9599; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9600; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9601; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9602; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9603; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9604; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9605; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9606; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9607; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9608; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9609; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9610; GFX10-WGP-NEXT:    buffer_gl0_inv
9611; GFX10-WGP-NEXT:    s_endpgm
9612;
9613; GFX10-CU-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
9614; GFX10-CU:       ; %bb.0: ; %entry
9615; GFX10-CU-NEXT:    s_clause 0x1
9616; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9617; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9618; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9619; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9620; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9621; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9622; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9623; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9624; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9625; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9626; GFX10-CU-NEXT:    s_endpgm
9627;
9628; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
9629; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9630; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9631; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9632; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9633; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9634; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9635; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9636; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9637; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9638; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9639; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9640; SKIP-CACHE-INV-NEXT:    s_endpgm
9641;
9642; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
9643; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9644; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9645; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9646; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9647; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9648; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9649; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9650; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9651;
9652; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
9653; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9654; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9655; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9656; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9657; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9658; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9659; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9660; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9661; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9662; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9663; GFX90A-TGSPLIT-NEXT:    s_endpgm
9664;
9665; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
9666; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9667; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9668; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9669; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9670; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9671; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9672; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9673; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9674;
9675; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
9676; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9677; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9678; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9679; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9680; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9681; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9682; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9683; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9684; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9685; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
9686; GFX940-TGSPLIT-NEXT:    s_endpgm
9687;
9688; GFX11-WGP-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
9689; GFX11-WGP:       ; %bb.0: ; %entry
9690; GFX11-WGP-NEXT:    s_clause 0x1
9691; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9692; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9693; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9694; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9695; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9696; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9697; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9698; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9699; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9700; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9701; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9702; GFX11-WGP-NEXT:    buffer_gl0_inv
9703; GFX11-WGP-NEXT:    s_endpgm
9704;
9705; GFX11-CU-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
9706; GFX11-CU:       ; %bb.0: ; %entry
9707; GFX11-CU-NEXT:    s_clause 0x1
9708; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9709; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9710; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9711; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9712; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9713; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9714; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9715; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9716; GFX11-CU-NEXT:    s_endpgm
9717    i32* %out, i32 %in, i32 %old) {
9718entry:
9719  %gep = getelementptr i32, i32* %out, i32 4
9720  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
9721  ret void
9722}
9723
9724define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_cmpxchg(
9725; GFX7-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
9726; GFX7:       ; %bb.0: ; %entry
9727; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9728; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9729; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9730; GFX7-NEXT:    s_add_u32 s0, s0, 16
9731; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9732; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9733; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9734; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9735; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9736; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9737; GFX7-NEXT:    s_endpgm
9738;
9739; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
9740; GFX10-WGP:       ; %bb.0: ; %entry
9741; GFX10-WGP-NEXT:    s_clause 0x1
9742; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9743; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9744; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9745; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9746; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9747; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9748; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9749; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9750; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9751; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9752; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9753; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9754; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9755; GFX10-WGP-NEXT:    buffer_gl0_inv
9756; GFX10-WGP-NEXT:    s_endpgm
9757;
9758; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
9759; GFX10-CU:       ; %bb.0: ; %entry
9760; GFX10-CU-NEXT:    s_clause 0x1
9761; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9762; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9763; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9764; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9765; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9766; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9767; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9768; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9769; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9770; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9771; GFX10-CU-NEXT:    s_endpgm
9772;
9773; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
9774; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9775; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9776; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9777; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9778; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9779; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9780; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9781; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9782; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9783; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9784; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9785; SKIP-CACHE-INV-NEXT:    s_endpgm
9786;
9787; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
9788; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9789; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9790; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9791; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9792; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9793; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9794; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9795; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9796;
9797; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
9798; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9799; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9800; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9801; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9802; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9803; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9804; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9805; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9806; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9807; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9808; GFX90A-TGSPLIT-NEXT:    s_endpgm
9809;
9810; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
9811; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9812; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9813; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9814; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9815; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9816; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9817; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9818; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9819;
9820; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
9821; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9822; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9823; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9824; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9825; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9826; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9827; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9828; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9829; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9830; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
9831; GFX940-TGSPLIT-NEXT:    s_endpgm
9832;
9833; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
9834; GFX11-WGP:       ; %bb.0: ; %entry
9835; GFX11-WGP-NEXT:    s_clause 0x1
9836; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9837; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9838; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9839; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9840; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9841; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9842; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9843; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9844; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9845; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9846; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9847; GFX11-WGP-NEXT:    buffer_gl0_inv
9848; GFX11-WGP-NEXT:    s_endpgm
9849;
9850; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
9851; GFX11-CU:       ; %bb.0: ; %entry
9852; GFX11-CU-NEXT:    s_clause 0x1
9853; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9854; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9855; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9856; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9857; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9858; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9859; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9860; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9861; GFX11-CU-NEXT:    s_endpgm
9862    i32* %out, i32 %in, i32 %old) {
9863entry:
9864  %gep = getelementptr i32, i32* %out, i32 4
9865  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
9866  ret void
9867}
9868
9869define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_cmpxchg(
9870; GFX7-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
9871; GFX7:       ; %bb.0: ; %entry
9872; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9873; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9874; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9875; GFX7-NEXT:    s_add_u32 s0, s0, 16
9876; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9877; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9878; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9879; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9880; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9881; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9882; GFX7-NEXT:    s_endpgm
9883;
9884; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
9885; GFX10-WGP:       ; %bb.0: ; %entry
9886; GFX10-WGP-NEXT:    s_clause 0x1
9887; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9888; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9889; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9890; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9891; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9892; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9893; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9894; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9895; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9896; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9897; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9898; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9899; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9900; GFX10-WGP-NEXT:    buffer_gl0_inv
9901; GFX10-WGP-NEXT:    s_endpgm
9902;
9903; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
9904; GFX10-CU:       ; %bb.0: ; %entry
9905; GFX10-CU-NEXT:    s_clause 0x1
9906; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9907; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9908; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9909; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9910; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9911; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9912; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9913; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9914; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9915; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9916; GFX10-CU-NEXT:    s_endpgm
9917;
9918; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
9919; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9920; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9921; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9922; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9923; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9924; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9925; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9926; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9927; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9928; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9929; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9930; SKIP-CACHE-INV-NEXT:    s_endpgm
9931;
9932; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
9933; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9934; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9935; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9936; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9937; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9938; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9939; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9940; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9941;
9942; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
9943; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9944; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9945; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9946; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9947; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9948; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9949; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9950; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9951; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9952; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9953; GFX90A-TGSPLIT-NEXT:    s_endpgm
9954;
9955; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
9956; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9957; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9958; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9959; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9960; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9961; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9962; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9963; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9964;
9965; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
9966; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9967; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9968; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9969; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9970; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9971; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9972; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9973; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9974; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9975; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
9976; GFX940-TGSPLIT-NEXT:    s_endpgm
9977;
9978; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
9979; GFX11-WGP:       ; %bb.0: ; %entry
9980; GFX11-WGP-NEXT:    s_clause 0x1
9981; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9982; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9983; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9984; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9985; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9986; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9987; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9988; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9989; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9990; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9991; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9992; GFX11-WGP-NEXT:    buffer_gl0_inv
9993; GFX11-WGP-NEXT:    s_endpgm
9994;
9995; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
9996; GFX11-CU:       ; %bb.0: ; %entry
9997; GFX11-CU-NEXT:    s_clause 0x1
9998; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9999; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10000; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10001; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10002; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10003; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10004; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10005; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10006; GFX11-CU-NEXT:    s_endpgm
10007    i32* %out, i32 %in, i32 %old) {
10008entry:
10009  %gep = getelementptr i32, i32* %out, i32 4
10010  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
10011  ret void
10012}
10013
10014define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_cmpxchg(
10015; GFX7-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
10016; GFX7:       ; %bb.0: ; %entry
10017; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10018; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10019; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10020; GFX7-NEXT:    s_add_u32 s0, s0, 16
10021; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10022; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10023; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10024; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10025; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10026; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10027; GFX7-NEXT:    s_endpgm
10028;
10029; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
10030; GFX10-WGP:       ; %bb.0: ; %entry
10031; GFX10-WGP-NEXT:    s_clause 0x1
10032; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10033; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10034; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10035; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10036; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10037; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10038; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10039; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10040; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10041; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10042; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10043; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10044; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10045; GFX10-WGP-NEXT:    buffer_gl0_inv
10046; GFX10-WGP-NEXT:    s_endpgm
10047;
10048; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
10049; GFX10-CU:       ; %bb.0: ; %entry
10050; GFX10-CU-NEXT:    s_clause 0x1
10051; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10052; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10053; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10054; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10055; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10056; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10057; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10058; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10059; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10060; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10061; GFX10-CU-NEXT:    s_endpgm
10062;
10063; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
10064; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10065; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10066; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10067; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10068; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10069; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10070; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10071; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10072; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10073; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10074; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10075; SKIP-CACHE-INV-NEXT:    s_endpgm
10076;
10077; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
10078; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10079; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10080; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10081; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10082; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10083; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10084; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10085; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10086;
10087; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
10088; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10089; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10090; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10091; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10092; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10093; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10094; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10095; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10096; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10097; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10098; GFX90A-TGSPLIT-NEXT:    s_endpgm
10099;
10100; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
10101; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10102; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10103; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10104; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10105; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10106; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10107; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10108; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10109;
10110; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
10111; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10112; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10113; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10114; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10115; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10116; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10117; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10118; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10119; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10120; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
10121; GFX940-TGSPLIT-NEXT:    s_endpgm
10122;
10123; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
10124; GFX11-WGP:       ; %bb.0: ; %entry
10125; GFX11-WGP-NEXT:    s_clause 0x1
10126; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10127; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10128; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10129; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10130; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10131; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10132; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10133; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10134; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10135; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10136; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10137; GFX11-WGP-NEXT:    buffer_gl0_inv
10138; GFX11-WGP-NEXT:    s_endpgm
10139;
10140; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
10141; GFX11-CU:       ; %bb.0: ; %entry
10142; GFX11-CU-NEXT:    s_clause 0x1
10143; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10144; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10145; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10146; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10147; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10148; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10149; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10150; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10151; GFX11-CU-NEXT:    s_endpgm
10152    i32* %out, i32 %in, i32 %old) {
10153entry:
10154  %gep = getelementptr i32, i32* %out, i32 4
10155  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic seq_cst
10156  ret void
10157}
10158
10159define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_cmpxchg(
10160; GFX7-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
10161; GFX7:       ; %bb.0: ; %entry
10162; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10163; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10164; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10165; GFX7-NEXT:    s_add_u32 s0, s0, 16
10166; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10167; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10168; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10169; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10170; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10171; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10172; GFX7-NEXT:    s_endpgm
10173;
10174; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
10175; GFX10-WGP:       ; %bb.0: ; %entry
10176; GFX10-WGP-NEXT:    s_clause 0x1
10177; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10178; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10179; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10180; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10181; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10182; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10183; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10184; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10185; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10186; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10187; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10188; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10189; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10190; GFX10-WGP-NEXT:    buffer_gl0_inv
10191; GFX10-WGP-NEXT:    s_endpgm
10192;
10193; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
10194; GFX10-CU:       ; %bb.0: ; %entry
10195; GFX10-CU-NEXT:    s_clause 0x1
10196; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10197; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10198; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10199; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10200; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10201; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10202; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10203; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10204; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10205; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10206; GFX10-CU-NEXT:    s_endpgm
10207;
10208; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
10209; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10210; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10211; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10212; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10213; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10214; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10215; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10216; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10217; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10218; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10219; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10220; SKIP-CACHE-INV-NEXT:    s_endpgm
10221;
10222; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
10223; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10224; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10225; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10226; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10227; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10228; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10229; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10230; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10231;
10232; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
10233; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10234; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10235; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10236; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10237; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10238; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10239; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10240; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10241; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10242; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10243; GFX90A-TGSPLIT-NEXT:    s_endpgm
10244;
10245; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
10246; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10247; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10248; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10249; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10250; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10251; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10252; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10253; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10254;
10255; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
10256; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10257; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10258; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10259; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10260; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10261; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10262; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10263; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10264; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10265; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
10266; GFX940-TGSPLIT-NEXT:    s_endpgm
10267;
10268; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
10269; GFX11-WGP:       ; %bb.0: ; %entry
10270; GFX11-WGP-NEXT:    s_clause 0x1
10271; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10272; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10273; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10274; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10275; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10276; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10277; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10278; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10279; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10280; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10281; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10282; GFX11-WGP-NEXT:    buffer_gl0_inv
10283; GFX11-WGP-NEXT:    s_endpgm
10284;
10285; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
10286; GFX11-CU:       ; %bb.0: ; %entry
10287; GFX11-CU-NEXT:    s_clause 0x1
10288; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10289; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10290; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10291; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10292; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10293; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10294; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10295; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10296; GFX11-CU-NEXT:    s_endpgm
10297    i32* %out, i32 %in, i32 %old) {
10298entry:
10299  %gep = getelementptr i32, i32* %out, i32 4
10300  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire seq_cst
10301  ret void
10302}
10303
10304define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_cmpxchg(
10305; GFX7-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
10306; GFX7:       ; %bb.0: ; %entry
10307; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10308; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10309; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10310; GFX7-NEXT:    s_add_u32 s0, s0, 16
10311; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10312; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10313; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10314; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10315; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10316; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10317; GFX7-NEXT:    s_endpgm
10318;
10319; GFX10-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
10320; GFX10-WGP:       ; %bb.0: ; %entry
10321; GFX10-WGP-NEXT:    s_clause 0x1
10322; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10323; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10324; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10325; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10326; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10327; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10328; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10329; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10330; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10331; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10332; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10333; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10334; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10335; GFX10-WGP-NEXT:    buffer_gl0_inv
10336; GFX10-WGP-NEXT:    s_endpgm
10337;
10338; GFX10-CU-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
10339; GFX10-CU:       ; %bb.0: ; %entry
10340; GFX10-CU-NEXT:    s_clause 0x1
10341; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10342; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10343; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10344; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10345; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10346; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10347; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10348; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10349; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10350; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10351; GFX10-CU-NEXT:    s_endpgm
10352;
10353; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
10354; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10355; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10356; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10357; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10358; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10359; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10360; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10361; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10362; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10363; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10364; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10365; SKIP-CACHE-INV-NEXT:    s_endpgm
10366;
10367; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
10368; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10369; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10370; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10371; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10372; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10373; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10374; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10375; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10376;
10377; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
10378; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10379; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10380; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10381; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10382; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10383; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10384; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10385; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10386; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10387; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10388; GFX90A-TGSPLIT-NEXT:    s_endpgm
10389;
10390; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
10391; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10392; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10393; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10394; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10395; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10396; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10397; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10398; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10399;
10400; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
10401; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10402; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10403; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10404; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10405; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10406; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10407; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10408; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10409; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10410; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
10411; GFX940-TGSPLIT-NEXT:    s_endpgm
10412;
10413; GFX11-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
10414; GFX11-WGP:       ; %bb.0: ; %entry
10415; GFX11-WGP-NEXT:    s_clause 0x1
10416; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10417; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10418; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10419; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10420; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10421; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10422; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10423; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10424; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10425; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10426; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10427; GFX11-WGP-NEXT:    buffer_gl0_inv
10428; GFX11-WGP-NEXT:    s_endpgm
10429;
10430; GFX11-CU-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
10431; GFX11-CU:       ; %bb.0: ; %entry
10432; GFX11-CU-NEXT:    s_clause 0x1
10433; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10434; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10435; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10436; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10437; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10438; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10439; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10440; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10441; GFX11-CU-NEXT:    s_endpgm
10442    i32* %out, i32 %in, i32 %old) {
10443entry:
10444  %gep = getelementptr i32, i32* %out, i32 4
10445  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release seq_cst
10446  ret void
10447}
10448
10449define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
10450; GFX7-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
10451; GFX7:       ; %bb.0: ; %entry
10452; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10453; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10454; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10455; GFX7-NEXT:    s_add_u32 s0, s0, 16
10456; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10457; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10458; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10459; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10460; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10461; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10462; GFX7-NEXT:    s_endpgm
10463;
10464; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
10465; GFX10-WGP:       ; %bb.0: ; %entry
10466; GFX10-WGP-NEXT:    s_clause 0x1
10467; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10468; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10469; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10470; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10471; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10472; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10473; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10474; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10475; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10476; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10477; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10478; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10479; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10480; GFX10-WGP-NEXT:    buffer_gl0_inv
10481; GFX10-WGP-NEXT:    s_endpgm
10482;
10483; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
10484; GFX10-CU:       ; %bb.0: ; %entry
10485; GFX10-CU-NEXT:    s_clause 0x1
10486; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10487; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10488; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10489; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10490; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10491; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10492; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10493; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10494; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10495; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10496; GFX10-CU-NEXT:    s_endpgm
10497;
10498; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
10499; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10500; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10501; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10502; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10503; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10504; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10505; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10506; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10507; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10508; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10509; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10510; SKIP-CACHE-INV-NEXT:    s_endpgm
10511;
10512; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
10513; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10514; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10515; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10516; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10517; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10518; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10519; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10520; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10521;
10522; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
10523; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10524; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10525; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10526; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10527; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10528; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10529; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10530; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10531; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10532; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10533; GFX90A-TGSPLIT-NEXT:    s_endpgm
10534;
10535; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
10536; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10537; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10538; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10539; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10540; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10541; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10542; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10543; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10544;
10545; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
10546; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10547; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10548; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10549; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10550; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10551; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10552; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10553; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10554; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10555; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
10556; GFX940-TGSPLIT-NEXT:    s_endpgm
10557;
10558; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
10559; GFX11-WGP:       ; %bb.0: ; %entry
10560; GFX11-WGP-NEXT:    s_clause 0x1
10561; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10562; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10563; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10564; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10565; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10566; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10567; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10568; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10569; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10570; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10571; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10572; GFX11-WGP-NEXT:    buffer_gl0_inv
10573; GFX11-WGP-NEXT:    s_endpgm
10574;
10575; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
10576; GFX11-CU:       ; %bb.0: ; %entry
10577; GFX11-CU-NEXT:    s_clause 0x1
10578; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10579; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10580; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10581; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10582; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10583; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10584; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10585; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10586; GFX11-CU-NEXT:    s_endpgm
10587    i32* %out, i32 %in, i32 %old) {
10588entry:
10589  %gep = getelementptr i32, i32* %out, i32 4
10590  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel seq_cst
10591  ret void
10592}
10593
10594define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
10595; GFX7-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
10596; GFX7:       ; %bb.0: ; %entry
10597; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10598; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10599; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10600; GFX7-NEXT:    s_add_u32 s0, s0, 16
10601; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10602; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10603; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10604; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10605; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10606; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10607; GFX7-NEXT:    s_endpgm
10608;
10609; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
10610; GFX10-WGP:       ; %bb.0: ; %entry
10611; GFX10-WGP-NEXT:    s_clause 0x1
10612; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10613; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10614; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10615; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10616; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10617; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10618; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10619; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10620; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10621; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10622; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10623; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10624; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10625; GFX10-WGP-NEXT:    buffer_gl0_inv
10626; GFX10-WGP-NEXT:    s_endpgm
10627;
10628; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
10629; GFX10-CU:       ; %bb.0: ; %entry
10630; GFX10-CU-NEXT:    s_clause 0x1
10631; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10632; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10633; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10634; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10635; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10636; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10637; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10638; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10639; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10640; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10641; GFX10-CU-NEXT:    s_endpgm
10642;
10643; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
10644; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10645; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10646; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10647; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10648; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10649; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10650; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10651; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10652; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10653; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10654; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10655; SKIP-CACHE-INV-NEXT:    s_endpgm
10656;
10657; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
10658; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10659; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10660; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10661; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10662; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10663; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10664; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10665; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10666;
10667; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
10668; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10669; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10670; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10671; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10672; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10673; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10674; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10675; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10676; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10677; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10678; GFX90A-TGSPLIT-NEXT:    s_endpgm
10679;
10680; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
10681; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10682; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10683; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10684; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10685; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10686; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10687; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10688; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10689;
10690; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
10691; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10692; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10693; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10694; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10695; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10696; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10697; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10698; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10699; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10700; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
10701; GFX940-TGSPLIT-NEXT:    s_endpgm
10702;
10703; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
10704; GFX11-WGP:       ; %bb.0: ; %entry
10705; GFX11-WGP-NEXT:    s_clause 0x1
10706; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10707; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10708; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10709; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10710; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10711; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10712; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10713; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10714; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10715; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10716; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10717; GFX11-WGP-NEXT:    buffer_gl0_inv
10718; GFX11-WGP-NEXT:    s_endpgm
10719;
10720; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
10721; GFX11-CU:       ; %bb.0: ; %entry
10722; GFX11-CU-NEXT:    s_clause 0x1
10723; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10724; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10725; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10726; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10727; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10728; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10729; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10730; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10731; GFX11-CU-NEXT:    s_endpgm
10732    i32* %out, i32 %in, i32 %old) {
10733entry:
10734  %gep = getelementptr i32, i32* %out, i32 4
10735  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
10736  ret void
10737}
10738
10739define amdgpu_kernel void @flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg(
10740; GFX7-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
10741; GFX7:       ; %bb.0: ; %entry
10742; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10743; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10744; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10745; GFX7-NEXT:    s_add_u32 s4, s0, 16
10746; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10747; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10748; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10749; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10750; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10751; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10752; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10753; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10754; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10755; GFX7-NEXT:    flat_store_dword v[0:1], v2
10756; GFX7-NEXT:    s_endpgm
10757;
10758; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
10759; GFX10-WGP:       ; %bb.0: ; %entry
10760; GFX10-WGP-NEXT:    s_clause 0x1
10761; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10762; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10763; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10764; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10765; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10766; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10767; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10768; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10769; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10770; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10771; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10772; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10773; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10774; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10775; GFX10-WGP-NEXT:    s_endpgm
10776;
10777; GFX10-CU-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
10778; GFX10-CU:       ; %bb.0: ; %entry
10779; GFX10-CU-NEXT:    s_clause 0x1
10780; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10781; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10782; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10783; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10784; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10785; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10786; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10787; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10788; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10789; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10790; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10791; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10792; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10793; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10794; GFX10-CU-NEXT:    s_endpgm
10795;
10796; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
10797; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10798; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10799; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10800; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10801; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10802; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10803; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10804; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10805; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10806; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10807; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10808; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10809; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10810; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10811; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10812; SKIP-CACHE-INV-NEXT:    s_endpgm
10813;
10814; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
10815; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10816; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10817; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10818; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10819; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10820; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10821; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10822; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10823; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10824; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10825;
10826; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
10827; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10828; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10829; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10830; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10831; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10832; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10833; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10834; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10835; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10836; GFX90A-TGSPLIT-NEXT:    s_endpgm
10837;
10838; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
10839; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10840; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10841; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10842; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10843; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10844; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10845; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10846; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10847; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10848; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10849;
10850; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
10851; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10852; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10853; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10854; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10855; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10856; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10857; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10858; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10859; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10860; GFX940-TGSPLIT-NEXT:    s_endpgm
10861;
10862; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
10863; GFX11-WGP:       ; %bb.0: ; %entry
10864; GFX11-WGP-NEXT:    s_clause 0x1
10865; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10866; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10867; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10868; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10869; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10870; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10871; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10872; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10873; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10874; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
10875; GFX11-WGP-NEXT:    s_endpgm
10876;
10877; GFX11-CU-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
10878; GFX11-CU:       ; %bb.0: ; %entry
10879; GFX11-CU-NEXT:    s_clause 0x1
10880; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10881; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10882; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10883; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10884; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10885; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10886; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10887; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10888; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10889; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
10890; GFX11-CU-NEXT:    s_endpgm
10891    i32* %out, i32 %in, i32 %old) {
10892entry:
10893  %gep = getelementptr i32, i32* %out, i32 4
10894  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic
10895  %val0 = extractvalue { i32, i1 } %val, 0
10896  store i32 %val0, i32* %out, align 4
10897  ret void
10898}
10899
10900define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
10901; GFX7-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
10902; GFX7:       ; %bb.0: ; %entry
10903; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10904; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10905; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10906; GFX7-NEXT:    s_add_u32 s4, s0, 16
10907; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10908; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10909; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10910; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10911; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10912; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10913; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10914; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10915; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10916; GFX7-NEXT:    flat_store_dword v[0:1], v2
10917; GFX7-NEXT:    s_endpgm
10918;
10919; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
10920; GFX10-WGP:       ; %bb.0: ; %entry
10921; GFX10-WGP-NEXT:    s_clause 0x1
10922; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10923; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10924; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10925; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10926; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10927; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10928; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10929; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10930; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10931; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10932; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10933; GFX10-WGP-NEXT:    buffer_gl0_inv
10934; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10935; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10936; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10937; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10938; GFX10-WGP-NEXT:    s_endpgm
10939;
10940; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
10941; GFX10-CU:       ; %bb.0: ; %entry
10942; GFX10-CU-NEXT:    s_clause 0x1
10943; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10944; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10945; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10946; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10947; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10948; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10949; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10950; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10951; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10952; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10953; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10954; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10955; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10956; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10957; GFX10-CU-NEXT:    s_endpgm
10958;
10959; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
10960; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10961; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10962; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10963; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10964; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10965; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10966; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10967; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10968; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10969; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10970; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10971; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10972; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10973; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10974; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10975; SKIP-CACHE-INV-NEXT:    s_endpgm
10976;
10977; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
10978; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10979; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10980; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10981; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10982; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10983; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10984; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10985; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10986; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10987; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10988;
10989; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
10990; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10991; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10992; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10993; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10994; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10995; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10996; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10997; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10998; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10999; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11000; GFX90A-TGSPLIT-NEXT:    s_endpgm
11001;
11002; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
11003; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11004; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11005; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11006; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11007; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11008; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11009; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11010; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11011; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11012; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11013;
11014; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
11015; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11016; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11017; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11018; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11019; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11020; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11021; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11022; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11023; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
11024; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11025; GFX940-TGSPLIT-NEXT:    s_endpgm
11026;
11027; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
11028; GFX11-WGP:       ; %bb.0: ; %entry
11029; GFX11-WGP-NEXT:    s_clause 0x1
11030; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11031; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11032; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11033; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11034; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11035; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11036; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11037; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11038; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11039; GFX11-WGP-NEXT:    buffer_gl0_inv
11040; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11041; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11042; GFX11-WGP-NEXT:    s_endpgm
11043;
11044; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
11045; GFX11-CU:       ; %bb.0: ; %entry
11046; GFX11-CU-NEXT:    s_clause 0x1
11047; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11048; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11049; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11050; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11051; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11052; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11053; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11054; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11055; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11056; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11057; GFX11-CU-NEXT:    s_endpgm
11058    i32* %out, i32 %in, i32 %old) {
11059entry:
11060  %gep = getelementptr i32, i32* %out, i32 4
11061  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
11062  %val0 = extractvalue { i32, i1 } %val, 0
11063  store i32 %val0, i32* %out, align 4
11064  ret void
11065}
11066
11067define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_ret_cmpxchg(
11068; GFX7-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
11069; GFX7:       ; %bb.0: ; %entry
11070; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11071; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11072; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11073; GFX7-NEXT:    s_add_u32 s4, s0, 16
11074; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11075; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11076; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11077; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11078; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11079; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11080; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11081; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11082; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11083; GFX7-NEXT:    flat_store_dword v[0:1], v2
11084; GFX7-NEXT:    s_endpgm
11085;
11086; GFX10-WGP-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
11087; GFX10-WGP:       ; %bb.0: ; %entry
11088; GFX10-WGP-NEXT:    s_clause 0x1
11089; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11090; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11091; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11092; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11093; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11094; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11095; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11096; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11097; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11098; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11099; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11100; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11101; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11102; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11103; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11104; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11105; GFX10-WGP-NEXT:    s_endpgm
11106;
11107; GFX10-CU-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
11108; GFX10-CU:       ; %bb.0: ; %entry
11109; GFX10-CU-NEXT:    s_clause 0x1
11110; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11111; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11112; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11113; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11114; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11115; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11116; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11117; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11118; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11119; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11120; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11121; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11122; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11123; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11124; GFX10-CU-NEXT:    s_endpgm
11125;
11126; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
11127; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11128; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11129; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11130; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11131; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11132; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11133; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11134; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11135; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11136; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11137; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11138; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11139; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11140; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11141; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11142; SKIP-CACHE-INV-NEXT:    s_endpgm
11143;
11144; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
11145; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11146; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11147; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11148; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11149; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11150; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11151; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11152; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11153; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11154; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11155;
11156; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
11157; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11158; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11159; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11160; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11161; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11162; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11163; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11164; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11165; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11166; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11167; GFX90A-TGSPLIT-NEXT:    s_endpgm
11168;
11169; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
11170; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11171; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11172; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11173; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11174; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11175; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11176; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11177; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11178; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11179; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11180;
11181; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
11182; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11183; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11184; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11185; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11186; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11187; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11188; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11189; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11190; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11191; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11192; GFX940-TGSPLIT-NEXT:    s_endpgm
11193;
11194; GFX11-WGP-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
11195; GFX11-WGP:       ; %bb.0: ; %entry
11196; GFX11-WGP-NEXT:    s_clause 0x1
11197; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11198; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11199; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11200; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11201; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11202; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11203; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11204; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11205; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11206; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11207; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11208; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11209; GFX11-WGP-NEXT:    s_endpgm
11210;
11211; GFX11-CU-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
11212; GFX11-CU:       ; %bb.0: ; %entry
11213; GFX11-CU-NEXT:    s_clause 0x1
11214; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11215; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11216; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11217; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11218; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11219; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11220; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11221; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11222; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11223; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11224; GFX11-CU-NEXT:    s_endpgm
11225    i32* %out, i32 %in, i32 %old) {
11226entry:
11227  %gep = getelementptr i32, i32* %out, i32 4
11228  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic
11229  %val0 = extractvalue { i32, i1 } %val, 0
11230  store i32 %val0, i32* %out, align 4
11231  ret void
11232}
11233
11234define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
11235; GFX7-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
11236; GFX7:       ; %bb.0: ; %entry
11237; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11238; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11239; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11240; GFX7-NEXT:    s_add_u32 s4, s0, 16
11241; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11242; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11243; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11244; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11245; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11246; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11247; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11248; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11249; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11250; GFX7-NEXT:    flat_store_dword v[0:1], v2
11251; GFX7-NEXT:    s_endpgm
11252;
11253; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
11254; GFX10-WGP:       ; %bb.0: ; %entry
11255; GFX10-WGP-NEXT:    s_clause 0x1
11256; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11257; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11258; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11259; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11260; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11261; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11262; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11263; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11264; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11265; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11266; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11267; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11268; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11269; GFX10-WGP-NEXT:    buffer_gl0_inv
11270; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11271; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11272; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11273; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11274; GFX10-WGP-NEXT:    s_endpgm
11275;
11276; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
11277; GFX10-CU:       ; %bb.0: ; %entry
11278; GFX10-CU-NEXT:    s_clause 0x1
11279; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11280; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11281; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11282; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11283; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11284; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11285; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11286; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11287; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11288; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11289; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11290; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11291; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11292; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11293; GFX10-CU-NEXT:    s_endpgm
11294;
11295; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
11296; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11297; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11298; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11299; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11300; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11301; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11302; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11303; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11304; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11305; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11306; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11307; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11308; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11309; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11310; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11311; SKIP-CACHE-INV-NEXT:    s_endpgm
11312;
11313; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
11314; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11315; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11316; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11317; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11318; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11319; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11320; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11321; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11322; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11323; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11324;
11325; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
11326; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11327; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11328; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11329; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11330; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11331; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11332; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11333; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11334; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11335; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11336; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11337; GFX90A-TGSPLIT-NEXT:    s_endpgm
11338;
11339; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
11340; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11341; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11342; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11343; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11344; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11345; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11346; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11347; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11348; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11349; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11350;
11351; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
11352; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11353; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11354; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11355; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11356; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11357; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11358; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11359; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11360; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11361; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
11362; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11363; GFX940-TGSPLIT-NEXT:    s_endpgm
11364;
11365; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
11366; GFX11-WGP:       ; %bb.0: ; %entry
11367; GFX11-WGP-NEXT:    s_clause 0x1
11368; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11369; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11370; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11371; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11372; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11373; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11374; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11375; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11376; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11377; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11378; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11379; GFX11-WGP-NEXT:    buffer_gl0_inv
11380; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11381; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11382; GFX11-WGP-NEXT:    s_endpgm
11383;
11384; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
11385; GFX11-CU:       ; %bb.0: ; %entry
11386; GFX11-CU-NEXT:    s_clause 0x1
11387; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11388; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11389; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11390; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11391; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11392; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11393; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11394; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11395; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11396; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11397; GFX11-CU-NEXT:    s_endpgm
11398    i32* %out, i32 %in, i32 %old) {
11399entry:
11400  %gep = getelementptr i32, i32* %out, i32 4
11401  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
11402  %val0 = extractvalue { i32, i1 } %val, 0
11403  store i32 %val0, i32* %out, align 4
11404  ret void
11405}
11406
11407define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
11408; GFX7-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
11409; GFX7:       ; %bb.0: ; %entry
11410; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11411; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11412; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11413; GFX7-NEXT:    s_add_u32 s4, s0, 16
11414; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11415; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11416; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11417; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11418; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11419; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11420; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11421; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11422; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11423; GFX7-NEXT:    flat_store_dword v[0:1], v2
11424; GFX7-NEXT:    s_endpgm
11425;
11426; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
11427; GFX10-WGP:       ; %bb.0: ; %entry
11428; GFX10-WGP-NEXT:    s_clause 0x1
11429; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11430; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11431; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11432; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11433; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11434; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11435; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11436; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11437; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11438; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11439; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11440; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11441; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11442; GFX10-WGP-NEXT:    buffer_gl0_inv
11443; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11444; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11445; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11446; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11447; GFX10-WGP-NEXT:    s_endpgm
11448;
11449; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
11450; GFX10-CU:       ; %bb.0: ; %entry
11451; GFX10-CU-NEXT:    s_clause 0x1
11452; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11453; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11454; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11455; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11456; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11457; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11458; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11459; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11460; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11461; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11462; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11463; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11464; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11465; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11466; GFX10-CU-NEXT:    s_endpgm
11467;
11468; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
11469; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11470; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11471; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11472; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11473; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11474; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11475; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11476; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11477; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11478; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11479; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11480; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11481; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11482; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11483; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11484; SKIP-CACHE-INV-NEXT:    s_endpgm
11485;
11486; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
11487; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11488; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11489; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11490; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11491; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11492; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11493; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11494; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11495; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11496; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11497;
11498; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
11499; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11500; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11501; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11502; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11503; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11504; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11505; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11506; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11507; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11508; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11509; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11510; GFX90A-TGSPLIT-NEXT:    s_endpgm
11511;
11512; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
11513; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11514; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11515; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11516; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11517; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11518; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11519; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11520; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11521; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11522; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11523;
11524; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
11525; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11526; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11527; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11528; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11529; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11530; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11531; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11532; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11533; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11534; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
11535; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11536; GFX940-TGSPLIT-NEXT:    s_endpgm
11537;
11538; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
11539; GFX11-WGP:       ; %bb.0: ; %entry
11540; GFX11-WGP-NEXT:    s_clause 0x1
11541; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11542; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11543; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11544; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11545; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11546; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11547; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11548; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11549; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11550; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11551; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11552; GFX11-WGP-NEXT:    buffer_gl0_inv
11553; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11554; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11555; GFX11-WGP-NEXT:    s_endpgm
11556;
11557; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
11558; GFX11-CU:       ; %bb.0: ; %entry
11559; GFX11-CU-NEXT:    s_clause 0x1
11560; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11561; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11562; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11563; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11564; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11565; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11566; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11567; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11568; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11569; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11570; GFX11-CU-NEXT:    s_endpgm
11571    i32* %out, i32 %in, i32 %old) {
11572entry:
11573  %gep = getelementptr i32, i32* %out, i32 4
11574  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
11575  %val0 = extractvalue { i32, i1 } %val, 0
11576  store i32 %val0, i32* %out, align 4
11577  ret void
11578}
11579
11580define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg(
11581; GFX7-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
11582; GFX7:       ; %bb.0: ; %entry
11583; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11584; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11585; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11586; GFX7-NEXT:    s_add_u32 s4, s0, 16
11587; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11588; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11589; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11590; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11591; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11592; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11593; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11594; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11595; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11596; GFX7-NEXT:    flat_store_dword v[0:1], v2
11597; GFX7-NEXT:    s_endpgm
11598;
11599; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
11600; GFX10-WGP:       ; %bb.0: ; %entry
11601; GFX10-WGP-NEXT:    s_clause 0x1
11602; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11603; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11604; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11605; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11606; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11607; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11608; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11609; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11610; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11611; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11612; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11613; GFX10-WGP-NEXT:    buffer_gl0_inv
11614; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11615; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11616; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11617; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11618; GFX10-WGP-NEXT:    s_endpgm
11619;
11620; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
11621; GFX10-CU:       ; %bb.0: ; %entry
11622; GFX10-CU-NEXT:    s_clause 0x1
11623; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11624; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11625; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11626; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11627; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11628; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11629; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11630; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11631; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11632; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11633; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11634; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11635; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11636; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11637; GFX10-CU-NEXT:    s_endpgm
11638;
11639; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
11640; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11641; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11642; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11643; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11644; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11645; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11646; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11647; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11648; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11649; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11650; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11651; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11652; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11653; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11654; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11655; SKIP-CACHE-INV-NEXT:    s_endpgm
11656;
11657; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
11658; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11659; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11660; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11661; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11662; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11663; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11664; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11665; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11666; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11667; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11668;
11669; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
11670; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11671; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11672; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11673; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11674; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11675; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11676; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11677; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11678; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11679; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11680; GFX90A-TGSPLIT-NEXT:    s_endpgm
11681;
11682; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
11683; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11684; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11685; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11686; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11687; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11688; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11689; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11690; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11691; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11692; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11693;
11694; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
11695; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11696; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11697; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11698; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11699; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11700; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11701; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11702; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11703; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
11704; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11705; GFX940-TGSPLIT-NEXT:    s_endpgm
11706;
11707; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
11708; GFX11-WGP:       ; %bb.0: ; %entry
11709; GFX11-WGP-NEXT:    s_clause 0x1
11710; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11711; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11712; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11713; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11714; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11715; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11716; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11717; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11718; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11719; GFX11-WGP-NEXT:    buffer_gl0_inv
11720; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11721; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11722; GFX11-WGP-NEXT:    s_endpgm
11723;
11724; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
11725; GFX11-CU:       ; %bb.0: ; %entry
11726; GFX11-CU-NEXT:    s_clause 0x1
11727; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11728; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11729; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11730; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11731; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11732; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11733; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11734; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11735; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11736; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11737; GFX11-CU-NEXT:    s_endpgm
11738    i32* %out, i32 %in, i32 %old) {
11739entry:
11740  %gep = getelementptr i32, i32* %out, i32 4
11741  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic acquire
11742  %val0 = extractvalue { i32, i1 } %val, 0
11743  store i32 %val0, i32* %out, align 4
11744  ret void
11745}
11746
11747define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_ret_cmpxchg(
11748; GFX7-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
11749; GFX7:       ; %bb.0: ; %entry
11750; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11751; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11752; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11753; GFX7-NEXT:    s_add_u32 s4, s0, 16
11754; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11755; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11756; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11757; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11758; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11759; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11760; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11761; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11762; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11763; GFX7-NEXT:    flat_store_dword v[0:1], v2
11764; GFX7-NEXT:    s_endpgm
11765;
11766; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
11767; GFX10-WGP:       ; %bb.0: ; %entry
11768; GFX10-WGP-NEXT:    s_clause 0x1
11769; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11770; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11771; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11772; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11773; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11774; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11775; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11776; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11777; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11778; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11779; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11780; GFX10-WGP-NEXT:    buffer_gl0_inv
11781; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11782; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11783; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11784; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11785; GFX10-WGP-NEXT:    s_endpgm
11786;
11787; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
11788; GFX10-CU:       ; %bb.0: ; %entry
11789; GFX10-CU-NEXT:    s_clause 0x1
11790; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11791; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11792; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11793; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11794; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11795; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11796; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11797; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11798; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11799; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11800; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11801; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11802; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11803; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11804; GFX10-CU-NEXT:    s_endpgm
11805;
11806; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
11807; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11808; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11809; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11810; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11811; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11812; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11813; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11814; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11815; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11816; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11817; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11818; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11819; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11820; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11821; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11822; SKIP-CACHE-INV-NEXT:    s_endpgm
11823;
11824; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
11825; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11826; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11827; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11828; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11829; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11830; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11831; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11832; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11833; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11834; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11835;
11836; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
11837; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11838; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11839; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11840; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11841; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11842; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11843; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11844; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11845; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11846; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11847; GFX90A-TGSPLIT-NEXT:    s_endpgm
11848;
11849; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
11850; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11851; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11852; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11853; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11854; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11855; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11856; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11857; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11858; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11859; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11860;
11861; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
11862; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11863; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11864; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11865; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11866; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11867; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11868; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11869; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11870; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
11871; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11872; GFX940-TGSPLIT-NEXT:    s_endpgm
11873;
11874; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
11875; GFX11-WGP:       ; %bb.0: ; %entry
11876; GFX11-WGP-NEXT:    s_clause 0x1
11877; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11878; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11879; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11880; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11881; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11882; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11883; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11884; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11885; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11886; GFX11-WGP-NEXT:    buffer_gl0_inv
11887; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11888; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11889; GFX11-WGP-NEXT:    s_endpgm
11890;
11891; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
11892; GFX11-CU:       ; %bb.0: ; %entry
11893; GFX11-CU-NEXT:    s_clause 0x1
11894; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11895; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11896; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11897; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11898; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11899; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11900; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11901; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11902; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11903; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11904; GFX11-CU-NEXT:    s_endpgm
11905    i32* %out, i32 %in, i32 %old) {
11906entry:
11907  %gep = getelementptr i32, i32* %out, i32 4
11908  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
11909  %val0 = extractvalue { i32, i1 } %val, 0
11910  store i32 %val0, i32* %out, align 4
11911  ret void
11912}
11913
11914define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_ret_cmpxchg(
11915; GFX7-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
11916; GFX7:       ; %bb.0: ; %entry
11917; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11918; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11919; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11920; GFX7-NEXT:    s_add_u32 s4, s0, 16
11921; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11922; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11923; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11924; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11925; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11926; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11927; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11928; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11929; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11930; GFX7-NEXT:    flat_store_dword v[0:1], v2
11931; GFX7-NEXT:    s_endpgm
11932;
11933; GFX10-WGP-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
11934; GFX10-WGP:       ; %bb.0: ; %entry
11935; GFX10-WGP-NEXT:    s_clause 0x1
11936; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11937; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11938; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11939; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11940; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11941; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11942; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11943; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11944; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11945; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11946; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11947; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11948; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11949; GFX10-WGP-NEXT:    buffer_gl0_inv
11950; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11951; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11952; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11953; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11954; GFX10-WGP-NEXT:    s_endpgm
11955;
11956; GFX10-CU-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
11957; GFX10-CU:       ; %bb.0: ; %entry
11958; GFX10-CU-NEXT:    s_clause 0x1
11959; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11960; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11961; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11962; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11963; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11964; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11965; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11966; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11967; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11968; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11969; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11970; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11971; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11972; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11973; GFX10-CU-NEXT:    s_endpgm
11974;
11975; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
11976; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11977; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11978; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11979; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11980; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11981; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11982; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11983; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11984; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11985; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11986; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11987; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11988; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11989; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11990; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11991; SKIP-CACHE-INV-NEXT:    s_endpgm
11992;
11993; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
11994; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11995; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11996; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11997; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11998; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11999; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12000; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12001; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12002; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12003; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12004;
12005; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
12006; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12007; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12008; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12009; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12010; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12011; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12012; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12013; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12014; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12015; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12016; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12017; GFX90A-TGSPLIT-NEXT:    s_endpgm
12018;
12019; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
12020; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12021; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12022; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12023; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12024; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12025; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12026; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12027; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12028; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12029; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12030;
12031; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
12032; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12033; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12034; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12035; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12036; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12037; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12038; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12039; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12040; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12041; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
12042; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12043; GFX940-TGSPLIT-NEXT:    s_endpgm
12044;
12045; GFX11-WGP-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
12046; GFX11-WGP:       ; %bb.0: ; %entry
12047; GFX11-WGP-NEXT:    s_clause 0x1
12048; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12049; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12050; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12051; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12052; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12053; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12054; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12055; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12056; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12057; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12058; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12059; GFX11-WGP-NEXT:    buffer_gl0_inv
12060; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12061; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12062; GFX11-WGP-NEXT:    s_endpgm
12063;
12064; GFX11-CU-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
12065; GFX11-CU:       ; %bb.0: ; %entry
12066; GFX11-CU-NEXT:    s_clause 0x1
12067; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12068; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12069; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12070; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12071; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12072; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12073; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12074; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12075; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12076; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12077; GFX11-CU-NEXT:    s_endpgm
12078    i32* %out, i32 %in, i32 %old) {
12079entry:
12080  %gep = getelementptr i32, i32* %out, i32 4
12081  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
12082  %val0 = extractvalue { i32, i1 } %val, 0
12083  store i32 %val0, i32* %out, align 4
12084  ret void
12085}
12086
12087define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
12088; GFX7-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
12089; GFX7:       ; %bb.0: ; %entry
12090; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12091; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12092; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12093; GFX7-NEXT:    s_add_u32 s4, s0, 16
12094; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12095; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12096; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12097; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12098; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12099; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12100; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12101; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12102; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12103; GFX7-NEXT:    flat_store_dword v[0:1], v2
12104; GFX7-NEXT:    s_endpgm
12105;
12106; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
12107; GFX10-WGP:       ; %bb.0: ; %entry
12108; GFX10-WGP-NEXT:    s_clause 0x1
12109; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12110; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12111; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12112; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12113; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12114; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12115; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12116; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12117; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12118; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12119; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12120; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12121; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12122; GFX10-WGP-NEXT:    buffer_gl0_inv
12123; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12124; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12125; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12126; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12127; GFX10-WGP-NEXT:    s_endpgm
12128;
12129; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
12130; GFX10-CU:       ; %bb.0: ; %entry
12131; GFX10-CU-NEXT:    s_clause 0x1
12132; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12133; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12134; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12135; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12136; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12137; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12138; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12139; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12140; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12141; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12142; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12143; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12144; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12145; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12146; GFX10-CU-NEXT:    s_endpgm
12147;
12148; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
12149; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12150; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12151; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12152; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12153; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12154; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12155; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12156; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12157; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12158; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12159; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12160; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12161; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12162; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12163; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12164; SKIP-CACHE-INV-NEXT:    s_endpgm
12165;
12166; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
12167; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12168; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12169; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12170; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12171; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12172; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12173; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12174; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12175; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12176; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12177;
12178; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
12179; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12180; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12181; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12182; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12183; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12184; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12185; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12186; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12187; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12188; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12189; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12190; GFX90A-TGSPLIT-NEXT:    s_endpgm
12191;
12192; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
12193; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12194; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12195; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12196; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12197; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12198; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12199; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12200; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12201; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12202; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12203;
12204; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
12205; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12206; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12207; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12208; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12209; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12210; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12211; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12212; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12213; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12214; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
12215; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12216; GFX940-TGSPLIT-NEXT:    s_endpgm
12217;
12218; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
12219; GFX11-WGP:       ; %bb.0: ; %entry
12220; GFX11-WGP-NEXT:    s_clause 0x1
12221; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12222; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12223; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12224; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12225; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12226; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12227; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12228; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12229; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12230; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12231; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12232; GFX11-WGP-NEXT:    buffer_gl0_inv
12233; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12234; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12235; GFX11-WGP-NEXT:    s_endpgm
12236;
12237; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
12238; GFX11-CU:       ; %bb.0: ; %entry
12239; GFX11-CU-NEXT:    s_clause 0x1
12240; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12241; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12242; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12243; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12244; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12245; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12246; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12247; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12248; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12249; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12250; GFX11-CU-NEXT:    s_endpgm
12251    i32* %out, i32 %in, i32 %old) {
12252entry:
12253  %gep = getelementptr i32, i32* %out, i32 4
12254  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
12255  %val0 = extractvalue { i32, i1 } %val, 0
12256  store i32 %val0, i32* %out, align 4
12257  ret void
12258}
12259
12260define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
12261; GFX7-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
12262; GFX7:       ; %bb.0: ; %entry
12263; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12264; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12265; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12266; GFX7-NEXT:    s_add_u32 s4, s0, 16
12267; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12268; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12269; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12270; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12271; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12272; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12273; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12274; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12275; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12276; GFX7-NEXT:    flat_store_dword v[0:1], v2
12277; GFX7-NEXT:    s_endpgm
12278;
12279; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
12280; GFX10-WGP:       ; %bb.0: ; %entry
12281; GFX10-WGP-NEXT:    s_clause 0x1
12282; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12283; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12284; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12285; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12286; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12287; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12288; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12289; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12290; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12291; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12292; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12293; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12294; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12295; GFX10-WGP-NEXT:    buffer_gl0_inv
12296; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12297; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12298; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12299; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12300; GFX10-WGP-NEXT:    s_endpgm
12301;
12302; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
12303; GFX10-CU:       ; %bb.0: ; %entry
12304; GFX10-CU-NEXT:    s_clause 0x1
12305; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12306; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12307; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12308; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12309; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12310; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12311; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12312; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12313; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12314; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12315; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12316; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12317; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12318; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12319; GFX10-CU-NEXT:    s_endpgm
12320;
12321; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
12322; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12323; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12324; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12325; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12326; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12327; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12328; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12329; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12330; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12331; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12332; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12333; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12334; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12335; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12336; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12337; SKIP-CACHE-INV-NEXT:    s_endpgm
12338;
12339; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
12340; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12341; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12342; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12343; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12344; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12345; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12346; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12347; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12348; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12349; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12350;
12351; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
12352; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12353; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12354; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12355; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12356; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12357; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12358; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12359; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12360; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12361; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12362; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12363; GFX90A-TGSPLIT-NEXT:    s_endpgm
12364;
12365; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
12366; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12367; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12368; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12369; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12370; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12371; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12372; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12373; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12374; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12375; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12376;
12377; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
12378; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12379; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12380; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12381; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12382; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12383; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12384; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12385; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12386; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12387; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
12388; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12389; GFX940-TGSPLIT-NEXT:    s_endpgm
12390;
12391; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
12392; GFX11-WGP:       ; %bb.0: ; %entry
12393; GFX11-WGP-NEXT:    s_clause 0x1
12394; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12395; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12396; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12397; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12398; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12399; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12400; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12401; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12402; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12403; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12404; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12405; GFX11-WGP-NEXT:    buffer_gl0_inv
12406; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12407; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12408; GFX11-WGP-NEXT:    s_endpgm
12409;
12410; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
12411; GFX11-CU:       ; %bb.0: ; %entry
12412; GFX11-CU-NEXT:    s_clause 0x1
12413; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12414; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12415; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12416; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12417; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12418; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12419; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12420; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12421; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12422; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12423; GFX11-CU-NEXT:    s_endpgm
12424    i32* %out, i32 %in, i32 %old) {
12425entry:
12426  %gep = getelementptr i32, i32* %out, i32 4
12427  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
12428  %val0 = extractvalue { i32, i1 } %val, 0
12429  store i32 %val0, i32* %out, align 4
12430  ret void
12431}
12432
12433define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg(
12434; GFX7-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
12435; GFX7:       ; %bb.0: ; %entry
12436; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12437; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12438; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12439; GFX7-NEXT:    s_add_u32 s4, s0, 16
12440; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12441; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12442; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12443; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12444; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12445; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12446; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12447; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12448; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12449; GFX7-NEXT:    flat_store_dword v[0:1], v2
12450; GFX7-NEXT:    s_endpgm
12451;
12452; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
12453; GFX10-WGP:       ; %bb.0: ; %entry
12454; GFX10-WGP-NEXT:    s_clause 0x1
12455; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12456; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12457; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12458; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12459; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12460; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12461; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12462; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12463; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12464; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12465; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12466; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12467; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12468; GFX10-WGP-NEXT:    buffer_gl0_inv
12469; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12470; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12471; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12472; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12473; GFX10-WGP-NEXT:    s_endpgm
12474;
12475; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
12476; GFX10-CU:       ; %bb.0: ; %entry
12477; GFX10-CU-NEXT:    s_clause 0x1
12478; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12479; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12480; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12481; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12482; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12483; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12484; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12485; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12486; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12487; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12488; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12489; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12490; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12491; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12492; GFX10-CU-NEXT:    s_endpgm
12493;
12494; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
12495; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12496; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12497; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12498; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12499; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12500; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12501; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12502; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12503; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12504; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12505; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12506; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12507; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12508; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12509; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12510; SKIP-CACHE-INV-NEXT:    s_endpgm
12511;
12512; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
12513; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12514; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12515; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12516; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12517; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12518; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12519; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12520; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12521; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12522; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12523;
12524; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
12525; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12526; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12527; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12528; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12529; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12530; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12531; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12532; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12533; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12534; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12535; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12536; GFX90A-TGSPLIT-NEXT:    s_endpgm
12537;
12538; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
12539; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12540; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12541; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12542; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12543; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12544; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12545; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12546; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12547; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12548; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12549;
12550; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
12551; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12552; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12553; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12554; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12555; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12556; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12557; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12558; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12559; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12560; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
12561; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12562; GFX940-TGSPLIT-NEXT:    s_endpgm
12563;
12564; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
12565; GFX11-WGP:       ; %bb.0: ; %entry
12566; GFX11-WGP-NEXT:    s_clause 0x1
12567; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12568; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12569; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12570; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12571; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12572; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12573; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12574; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12575; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12576; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12577; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12578; GFX11-WGP-NEXT:    buffer_gl0_inv
12579; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12580; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12581; GFX11-WGP-NEXT:    s_endpgm
12582;
12583; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
12584; GFX11-CU:       ; %bb.0: ; %entry
12585; GFX11-CU-NEXT:    s_clause 0x1
12586; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12587; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12588; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12589; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12590; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12591; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12592; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12593; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12594; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12595; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12596; GFX11-CU-NEXT:    s_endpgm
12597    i32* %out, i32 %in, i32 %old) {
12598entry:
12599  %gep = getelementptr i32, i32* %out, i32 4
12600  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic seq_cst
12601  %val0 = extractvalue { i32, i1 } %val, 0
12602  store i32 %val0, i32* %out, align 4
12603  ret void
12604}
12605
12606define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
12607; GFX7-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
12608; GFX7:       ; %bb.0: ; %entry
12609; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12610; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12611; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12612; GFX7-NEXT:    s_add_u32 s4, s0, 16
12613; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12614; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12615; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12616; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12617; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12618; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12619; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12620; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12621; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12622; GFX7-NEXT:    flat_store_dword v[0:1], v2
12623; GFX7-NEXT:    s_endpgm
12624;
12625; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
12626; GFX10-WGP:       ; %bb.0: ; %entry
12627; GFX10-WGP-NEXT:    s_clause 0x1
12628; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12629; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12630; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12631; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12632; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12633; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12634; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12635; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12636; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12637; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12638; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12639; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12640; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12641; GFX10-WGP-NEXT:    buffer_gl0_inv
12642; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12643; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12644; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12645; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12646; GFX10-WGP-NEXT:    s_endpgm
12647;
12648; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
12649; GFX10-CU:       ; %bb.0: ; %entry
12650; GFX10-CU-NEXT:    s_clause 0x1
12651; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12652; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12653; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12654; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12655; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12656; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12657; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12658; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12659; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12660; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12661; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12662; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12663; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12664; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12665; GFX10-CU-NEXT:    s_endpgm
12666;
12667; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
12668; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12669; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12670; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12671; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12672; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12673; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12674; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12675; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12676; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12677; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12678; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12679; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12680; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12681; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12682; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12683; SKIP-CACHE-INV-NEXT:    s_endpgm
12684;
12685; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
12686; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12687; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12688; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12689; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12690; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12691; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12692; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12693; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12694; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12695; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12696;
12697; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
12698; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12699; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12700; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12701; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12702; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12703; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12704; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12705; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12706; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12707; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12708; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12709; GFX90A-TGSPLIT-NEXT:    s_endpgm
12710;
12711; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
12712; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12713; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12714; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12715; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12716; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12717; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12718; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12719; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12720; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12721; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12722;
12723; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
12724; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12725; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12726; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12727; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12728; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12729; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12730; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12731; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12732; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12733; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
12734; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12735; GFX940-TGSPLIT-NEXT:    s_endpgm
12736;
12737; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
12738; GFX11-WGP:       ; %bb.0: ; %entry
12739; GFX11-WGP-NEXT:    s_clause 0x1
12740; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12741; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12742; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12743; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12744; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12745; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12746; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12747; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12748; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12749; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12750; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12751; GFX11-WGP-NEXT:    buffer_gl0_inv
12752; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12753; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12754; GFX11-WGP-NEXT:    s_endpgm
12755;
12756; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
12757; GFX11-CU:       ; %bb.0: ; %entry
12758; GFX11-CU-NEXT:    s_clause 0x1
12759; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12760; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12761; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12762; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12763; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12764; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12765; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12766; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12767; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12768; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12769; GFX11-CU-NEXT:    s_endpgm
12770    i32* %out, i32 %in, i32 %old) {
12771entry:
12772  %gep = getelementptr i32, i32* %out, i32 4
12773  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire seq_cst
12774  %val0 = extractvalue { i32, i1 } %val, 0
12775  store i32 %val0, i32* %out, align 4
12776  ret void
12777}
12778
12779define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_ret_cmpxchg(
12780; GFX7-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
12781; GFX7:       ; %bb.0: ; %entry
12782; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12783; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12784; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12785; GFX7-NEXT:    s_add_u32 s4, s0, 16
12786; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12787; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12788; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12789; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12790; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12791; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12792; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12793; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12794; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12795; GFX7-NEXT:    flat_store_dword v[0:1], v2
12796; GFX7-NEXT:    s_endpgm
12797;
12798; GFX10-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
12799; GFX10-WGP:       ; %bb.0: ; %entry
12800; GFX10-WGP-NEXT:    s_clause 0x1
12801; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12802; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12803; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12804; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12805; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12806; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12807; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12808; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12809; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12810; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12811; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12812; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12813; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12814; GFX10-WGP-NEXT:    buffer_gl0_inv
12815; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12816; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12817; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12818; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12819; GFX10-WGP-NEXT:    s_endpgm
12820;
12821; GFX10-CU-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
12822; GFX10-CU:       ; %bb.0: ; %entry
12823; GFX10-CU-NEXT:    s_clause 0x1
12824; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12825; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12826; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12827; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12828; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12829; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12830; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12831; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12832; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12833; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12834; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12835; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12836; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12837; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12838; GFX10-CU-NEXT:    s_endpgm
12839;
12840; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
12841; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12842; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12843; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12844; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12845; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12846; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12847; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12848; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12849; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12850; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12851; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12852; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12853; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12854; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12855; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12856; SKIP-CACHE-INV-NEXT:    s_endpgm
12857;
12858; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
12859; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12860; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12861; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12862; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12863; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12864; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12865; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12866; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12867; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12868; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12869;
12870; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
12871; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12872; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12873; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12874; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12875; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12876; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12877; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12878; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12879; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12880; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12881; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12882; GFX90A-TGSPLIT-NEXT:    s_endpgm
12883;
12884; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
12885; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12886; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12887; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12888; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12889; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12890; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12891; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12892; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12893; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12894; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12895;
12896; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
12897; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12898; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12899; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12900; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12901; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12902; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12903; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12904; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12905; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12906; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
12907; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12908; GFX940-TGSPLIT-NEXT:    s_endpgm
12909;
12910; GFX11-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
12911; GFX11-WGP:       ; %bb.0: ; %entry
12912; GFX11-WGP-NEXT:    s_clause 0x1
12913; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12914; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12915; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12916; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12917; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12918; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12919; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12920; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12921; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12922; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12923; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12924; GFX11-WGP-NEXT:    buffer_gl0_inv
12925; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12926; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12927; GFX11-WGP-NEXT:    s_endpgm
12928;
12929; GFX11-CU-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
12930; GFX11-CU:       ; %bb.0: ; %entry
12931; GFX11-CU-NEXT:    s_clause 0x1
12932; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12933; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12934; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12935; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12936; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12937; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12938; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12939; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12940; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12941; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12942; GFX11-CU-NEXT:    s_endpgm
12943    i32* %out, i32 %in, i32 %old) {
12944entry:
12945  %gep = getelementptr i32, i32* %out, i32 4
12946  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release seq_cst
12947  %val0 = extractvalue { i32, i1 } %val, 0
12948  store i32 %val0, i32* %out, align 4
12949  ret void
12950}
12951
12952define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
12953; GFX7-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
12954; GFX7:       ; %bb.0: ; %entry
12955; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12956; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12957; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12958; GFX7-NEXT:    s_add_u32 s4, s0, 16
12959; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12960; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12961; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12962; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12963; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12964; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12965; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12966; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12967; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12968; GFX7-NEXT:    flat_store_dword v[0:1], v2
12969; GFX7-NEXT:    s_endpgm
12970;
12971; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
12972; GFX10-WGP:       ; %bb.0: ; %entry
12973; GFX10-WGP-NEXT:    s_clause 0x1
12974; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12975; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12976; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12977; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12978; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12979; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12980; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12981; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12982; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12983; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12984; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12985; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12986; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12987; GFX10-WGP-NEXT:    buffer_gl0_inv
12988; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12989; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12990; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12991; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12992; GFX10-WGP-NEXT:    s_endpgm
12993;
12994; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
12995; GFX10-CU:       ; %bb.0: ; %entry
12996; GFX10-CU-NEXT:    s_clause 0x1
12997; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12998; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12999; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13000; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13001; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13002; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13003; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13004; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13005; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13006; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13007; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13008; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13009; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
13010; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13011; GFX10-CU-NEXT:    s_endpgm
13012;
13013; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
13014; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13015; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13016; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13017; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13018; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13019; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13020; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13021; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13022; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13023; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13024; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13025; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13026; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13027; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
13028; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13029; SKIP-CACHE-INV-NEXT:    s_endpgm
13030;
13031; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
13032; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13033; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13034; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13035; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13036; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13037; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13038; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13039; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
13040; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13041; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13042;
13043; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
13044; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13045; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13046; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13047; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13048; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13049; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13050; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13051; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13052; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13053; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13054; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13055; GFX90A-TGSPLIT-NEXT:    s_endpgm
13056;
13057; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
13058; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13059; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13060; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13061; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13062; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13063; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13064; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13065; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
13066; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13067; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13068;
13069; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
13070; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13071; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13072; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13073; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13074; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13075; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13076; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13077; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13078; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13079; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
13080; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13081; GFX940-TGSPLIT-NEXT:    s_endpgm
13082;
13083; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
13084; GFX11-WGP:       ; %bb.0: ; %entry
13085; GFX11-WGP-NEXT:    s_clause 0x1
13086; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13087; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13088; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13089; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
13090; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
13091; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
13092; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
13093; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13094; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13095; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13096; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13097; GFX11-WGP-NEXT:    buffer_gl0_inv
13098; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13099; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13100; GFX11-WGP-NEXT:    s_endpgm
13101;
13102; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
13103; GFX11-CU:       ; %bb.0: ; %entry
13104; GFX11-CU-NEXT:    s_clause 0x1
13105; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13106; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13107; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13108; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
13109; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
13110; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
13111; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
13112; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13113; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
13114; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13115; GFX11-CU-NEXT:    s_endpgm
13116    i32* %out, i32 %in, i32 %old) {
13117entry:
13118  %gep = getelementptr i32, i32* %out, i32 4
13119  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel seq_cst
13120  %val0 = extractvalue { i32, i1 } %val, 0
13121  store i32 %val0, i32* %out, align 4
13122  ret void
13123}
13124
13125define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
13126; GFX7-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
13127; GFX7:       ; %bb.0: ; %entry
13128; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13129; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13130; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13131; GFX7-NEXT:    s_add_u32 s4, s0, 16
13132; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13133; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13134; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13135; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13136; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13137; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13138; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13139; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13140; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
13141; GFX7-NEXT:    flat_store_dword v[0:1], v2
13142; GFX7-NEXT:    s_endpgm
13143;
13144; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
13145; GFX10-WGP:       ; %bb.0: ; %entry
13146; GFX10-WGP-NEXT:    s_clause 0x1
13147; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13148; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13149; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13150; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13151; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13152; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13153; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13154; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13155; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13156; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13157; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13158; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13159; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13160; GFX10-WGP-NEXT:    buffer_gl0_inv
13161; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13162; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13163; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13164; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13165; GFX10-WGP-NEXT:    s_endpgm
13166;
13167; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
13168; GFX10-CU:       ; %bb.0: ; %entry
13169; GFX10-CU-NEXT:    s_clause 0x1
13170; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13171; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13172; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13173; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13174; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13175; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13176; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13177; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13178; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13179; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13180; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13181; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13182; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
13183; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13184; GFX10-CU-NEXT:    s_endpgm
13185;
13186; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
13187; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13188; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13189; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13190; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13191; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13192; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13193; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13194; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13195; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13196; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13197; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13198; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13199; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13200; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
13201; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13202; SKIP-CACHE-INV-NEXT:    s_endpgm
13203;
13204; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
13205; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13206; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13207; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13208; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13209; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13210; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13211; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13212; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
13213; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13214; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13215;
13216; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
13217; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13218; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13219; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13220; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13221; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13222; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13223; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13224; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13225; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13226; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13227; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13228; GFX90A-TGSPLIT-NEXT:    s_endpgm
13229;
13230; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
13231; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13232; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13233; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13234; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13235; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13236; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13237; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13238; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
13239; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13240; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13241;
13242; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
13243; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13244; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13245; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13246; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13247; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13248; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13249; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13250; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13251; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13252; GFX940-TGSPLIT-NEXT:    buffer_inv sc0
13253; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13254; GFX940-TGSPLIT-NEXT:    s_endpgm
13255;
13256; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
13257; GFX11-WGP:       ; %bb.0: ; %entry
13258; GFX11-WGP-NEXT:    s_clause 0x1
13259; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13260; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13261; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13262; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
13263; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
13264; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
13265; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
13266; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13267; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13268; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13269; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13270; GFX11-WGP-NEXT:    buffer_gl0_inv
13271; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13272; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13273; GFX11-WGP-NEXT:    s_endpgm
13274;
13275; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
13276; GFX11-CU:       ; %bb.0: ; %entry
13277; GFX11-CU-NEXT:    s_clause 0x1
13278; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13279; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13280; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13281; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
13282; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
13283; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
13284; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
13285; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13286; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
13287; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13288; GFX11-CU-NEXT:    s_endpgm
13289    i32* %out, i32 %in, i32 %old) {
13290entry:
13291  %gep = getelementptr i32, i32* %out, i32 4
13292  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
13293  %val0 = extractvalue { i32, i1 } %val, 0
13294  store i32 %val0, i32* %out, align 4
13295  ret void
13296}
13297
13298