1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
8; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
9; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
10; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-WGP %s
11; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-CU %s
12
13define amdgpu_kernel void @flat_singlethread_unordered_load(
14; GFX7-LABEL: flat_singlethread_unordered_load:
15; GFX7:       ; %bb.0: ; %entry
16; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
17; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
18; GFX7-NEXT:    v_mov_b32_e32 v0, s0
19; GFX7-NEXT:    v_mov_b32_e32 v1, s1
20; GFX7-NEXT:    flat_load_dword v2, v[0:1]
21; GFX7-NEXT:    v_mov_b32_e32 v0, s2
22; GFX7-NEXT:    v_mov_b32_e32 v1, s3
23; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
24; GFX7-NEXT:    flat_store_dword v[0:1], v2
25; GFX7-NEXT:    s_endpgm
26;
27; GFX10-WGP-LABEL: flat_singlethread_unordered_load:
28; GFX10-WGP:       ; %bb.0: ; %entry
29; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
30; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
31; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
32; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
33; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
34; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
35; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
36; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
37; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
38; GFX10-WGP-NEXT:    s_endpgm
39;
40; GFX10-CU-LABEL: flat_singlethread_unordered_load:
41; GFX10-CU:       ; %bb.0: ; %entry
42; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
43; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
44; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
45; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
46; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
47; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
48; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
49; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
50; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
51; GFX10-CU-NEXT:    s_endpgm
52;
53; SKIP-CACHE-INV-LABEL: flat_singlethread_unordered_load:
54; SKIP-CACHE-INV:       ; %bb.0: ; %entry
55; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
56; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
57; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
58; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
59; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
60; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
61; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
62; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
63; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
64; SKIP-CACHE-INV-NEXT:    s_endpgm
65;
66; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_unordered_load:
67; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
68; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
69; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
70; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
71; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
72; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
73; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
74; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
75; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
76; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
77; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
78;
79; GFX90A-TGSPLIT-LABEL: flat_singlethread_unordered_load:
80; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
81; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
82; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
83; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
84; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
85; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
86; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
87; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
88; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
89; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
90; GFX90A-TGSPLIT-NEXT:    s_endpgm
91;
92; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_unordered_load:
93; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
94; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
95; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
96; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
97; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
98; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
99; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
100; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
101; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
102; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
103; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
104;
105; GFX940-TGSPLIT-LABEL: flat_singlethread_unordered_load:
106; GFX940-TGSPLIT:       ; %bb.0: ; %entry
107; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
108; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
109; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
110; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
111; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
112; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
113; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
114; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
115; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
116; GFX940-TGSPLIT-NEXT:    s_endpgm
117;
118; GFX11-WGP-LABEL: flat_singlethread_unordered_load:
119; GFX11-WGP:       ; %bb.0: ; %entry
120; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
121; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
122; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
123; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
124; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
125; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
126; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
127; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
128; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
129; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
130; GFX11-WGP-NEXT:    s_endpgm
131;
132; GFX11-CU-LABEL: flat_singlethread_unordered_load:
133; GFX11-CU:       ; %bb.0: ; %entry
134; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
135; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
136; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
137; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
138; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
139; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
140; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
141; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
142; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
143; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
144; GFX11-CU-NEXT:    s_endpgm
145    i32* %in, i32* %out) {
146entry:
147  %val = load atomic i32, i32* %in syncscope("singlethread") unordered, align 4
148  store i32 %val, i32* %out
149  ret void
150}
151
152define amdgpu_kernel void @flat_singlethread_monotonic_load(
153; GFX7-LABEL: flat_singlethread_monotonic_load:
154; GFX7:       ; %bb.0: ; %entry
155; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
156; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
157; GFX7-NEXT:    v_mov_b32_e32 v0, s0
158; GFX7-NEXT:    v_mov_b32_e32 v1, s1
159; GFX7-NEXT:    flat_load_dword v2, v[0:1]
160; GFX7-NEXT:    v_mov_b32_e32 v0, s2
161; GFX7-NEXT:    v_mov_b32_e32 v1, s3
162; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
163; GFX7-NEXT:    flat_store_dword v[0:1], v2
164; GFX7-NEXT:    s_endpgm
165;
166; GFX10-WGP-LABEL: flat_singlethread_monotonic_load:
167; GFX10-WGP:       ; %bb.0: ; %entry
168; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
169; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
170; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
171; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
172; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
173; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
174; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
175; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
176; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
177; GFX10-WGP-NEXT:    s_endpgm
178;
179; GFX10-CU-LABEL: flat_singlethread_monotonic_load:
180; GFX10-CU:       ; %bb.0: ; %entry
181; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
182; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
183; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
184; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
185; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
186; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
187; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
188; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
189; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
190; GFX10-CU-NEXT:    s_endpgm
191;
192; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_load:
193; SKIP-CACHE-INV:       ; %bb.0: ; %entry
194; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
195; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
196; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
197; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
198; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
199; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
200; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
201; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
202; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
203; SKIP-CACHE-INV-NEXT:    s_endpgm
204;
205; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_load:
206; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
207; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
208; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
209; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
210; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
211; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
212; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
213; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
214; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
215; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
216; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
217;
218; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_load:
219; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
220; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
221; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
222; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
223; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
224; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
225; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
226; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
227; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
228; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
229; GFX90A-TGSPLIT-NEXT:    s_endpgm
230;
231; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_load:
232; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
233; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
234; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
235; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
236; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
237; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
238; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
239; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
240; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
241; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
242; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
243;
244; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_load:
245; GFX940-TGSPLIT:       ; %bb.0: ; %entry
246; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
247; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
248; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
249; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
250; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
251; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
252; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
253; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
254; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
255; GFX940-TGSPLIT-NEXT:    s_endpgm
256;
257; GFX11-WGP-LABEL: flat_singlethread_monotonic_load:
258; GFX11-WGP:       ; %bb.0: ; %entry
259; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
260; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
261; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
262; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
263; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
264; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
265; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
266; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
267; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
268; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
269; GFX11-WGP-NEXT:    s_endpgm
270;
271; GFX11-CU-LABEL: flat_singlethread_monotonic_load:
272; GFX11-CU:       ; %bb.0: ; %entry
273; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
274; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
275; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
276; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
277; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
278; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
279; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
280; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
281; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
282; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
283; GFX11-CU-NEXT:    s_endpgm
284    i32* %in, i32* %out) {
285entry:
286  %val = load atomic i32, i32* %in syncscope("singlethread") monotonic, align 4
287  store i32 %val, i32* %out
288  ret void
289}
290
291define amdgpu_kernel void @flat_singlethread_acquire_load(
292; GFX7-LABEL: flat_singlethread_acquire_load:
293; GFX7:       ; %bb.0: ; %entry
294; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
295; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
296; GFX7-NEXT:    v_mov_b32_e32 v0, s0
297; GFX7-NEXT:    v_mov_b32_e32 v1, s1
298; GFX7-NEXT:    flat_load_dword v2, v[0:1]
299; GFX7-NEXT:    v_mov_b32_e32 v0, s2
300; GFX7-NEXT:    v_mov_b32_e32 v1, s3
301; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
302; GFX7-NEXT:    flat_store_dword v[0:1], v2
303; GFX7-NEXT:    s_endpgm
304;
305; GFX10-WGP-LABEL: flat_singlethread_acquire_load:
306; GFX10-WGP:       ; %bb.0: ; %entry
307; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
308; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
309; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
310; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
311; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
312; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
313; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
314; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
315; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
316; GFX10-WGP-NEXT:    s_endpgm
317;
318; GFX10-CU-LABEL: flat_singlethread_acquire_load:
319; GFX10-CU:       ; %bb.0: ; %entry
320; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
321; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
322; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
323; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
324; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
325; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
326; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
327; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
328; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
329; GFX10-CU-NEXT:    s_endpgm
330;
331; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_load:
332; SKIP-CACHE-INV:       ; %bb.0: ; %entry
333; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
334; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
335; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
336; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
337; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
338; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
339; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
340; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
341; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
342; SKIP-CACHE-INV-NEXT:    s_endpgm
343;
344; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_load:
345; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
346; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
347; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
348; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
349; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
350; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
351; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
352; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
353; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
354; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
355; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
356;
357; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_load:
358; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
359; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
360; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
361; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
362; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
363; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
364; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
365; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
366; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
367; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
368; GFX90A-TGSPLIT-NEXT:    s_endpgm
369;
370; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_load:
371; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
372; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
373; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
374; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
375; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
376; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
377; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
378; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
379; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
380; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
381; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
382;
383; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_load:
384; GFX940-TGSPLIT:       ; %bb.0: ; %entry
385; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
386; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
387; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
388; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
389; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
390; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
391; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
392; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
393; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
394; GFX940-TGSPLIT-NEXT:    s_endpgm
395;
396; GFX11-WGP-LABEL: flat_singlethread_acquire_load:
397; GFX11-WGP:       ; %bb.0: ; %entry
398; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
399; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
400; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
401; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
402; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
403; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
404; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
405; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
406; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
407; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
408; GFX11-WGP-NEXT:    s_endpgm
409;
410; GFX11-CU-LABEL: flat_singlethread_acquire_load:
411; GFX11-CU:       ; %bb.0: ; %entry
412; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
413; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
414; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
415; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
416; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
417; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
418; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
419; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
420; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
421; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
422; GFX11-CU-NEXT:    s_endpgm
423    i32* %in, i32* %out) {
424entry:
425  %val = load atomic i32, i32* %in syncscope("singlethread") acquire, align 4
426  store i32 %val, i32* %out
427  ret void
428}
429
430define amdgpu_kernel void @flat_singlethread_seq_cst_load(
431; GFX7-LABEL: flat_singlethread_seq_cst_load:
432; GFX7:       ; %bb.0: ; %entry
433; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
434; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
435; GFX7-NEXT:    v_mov_b32_e32 v0, s0
436; GFX7-NEXT:    v_mov_b32_e32 v1, s1
437; GFX7-NEXT:    flat_load_dword v2, v[0:1]
438; GFX7-NEXT:    v_mov_b32_e32 v0, s2
439; GFX7-NEXT:    v_mov_b32_e32 v1, s3
440; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
441; GFX7-NEXT:    flat_store_dword v[0:1], v2
442; GFX7-NEXT:    s_endpgm
443;
444; GFX10-WGP-LABEL: flat_singlethread_seq_cst_load:
445; GFX10-WGP:       ; %bb.0: ; %entry
446; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
447; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
448; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
449; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
450; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
451; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
452; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
453; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
454; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
455; GFX10-WGP-NEXT:    s_endpgm
456;
457; GFX10-CU-LABEL: flat_singlethread_seq_cst_load:
458; GFX10-CU:       ; %bb.0: ; %entry
459; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
460; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
461; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
462; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
463; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
464; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
465; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
466; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
467; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
468; GFX10-CU-NEXT:    s_endpgm
469;
470; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_load:
471; SKIP-CACHE-INV:       ; %bb.0: ; %entry
472; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
473; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
474; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
475; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
476; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
477; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
478; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
479; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
480; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
481; SKIP-CACHE-INV-NEXT:    s_endpgm
482;
483; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_load:
484; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
485; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
486; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
487; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
488; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
489; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
490; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
491; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
492; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
493; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
494; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
495;
496; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_load:
497; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
498; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
499; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
500; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
501; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
502; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
503; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
504; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
505; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
506; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
507; GFX90A-TGSPLIT-NEXT:    s_endpgm
508;
509; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_load:
510; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
511; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
512; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
513; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
514; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
515; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
516; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
517; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
518; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
519; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
520; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
521;
522; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_load:
523; GFX940-TGSPLIT:       ; %bb.0: ; %entry
524; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
525; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
526; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
527; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
528; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
529; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
530; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
531; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
532; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
533; GFX940-TGSPLIT-NEXT:    s_endpgm
534;
535; GFX11-WGP-LABEL: flat_singlethread_seq_cst_load:
536; GFX11-WGP:       ; %bb.0: ; %entry
537; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
538; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
539; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
540; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
541; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
542; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
543; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
544; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
545; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
546; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
547; GFX11-WGP-NEXT:    s_endpgm
548;
549; GFX11-CU-LABEL: flat_singlethread_seq_cst_load:
550; GFX11-CU:       ; %bb.0: ; %entry
551; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
552; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
553; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
554; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
555; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
556; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
557; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
558; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
559; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
560; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
561; GFX11-CU-NEXT:    s_endpgm
562    i32* %in, i32* %out) {
563entry:
564  %val = load atomic i32, i32* %in syncscope("singlethread") seq_cst, align 4
565  store i32 %val, i32* %out
566  ret void
567}
568
569define amdgpu_kernel void @flat_singlethread_unordered_store(
570; GFX7-LABEL: flat_singlethread_unordered_store:
571; GFX7:       ; %bb.0: ; %entry
572; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
573; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
574; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
575; GFX7-NEXT:    v_mov_b32_e32 v0, s0
576; GFX7-NEXT:    v_mov_b32_e32 v1, s1
577; GFX7-NEXT:    v_mov_b32_e32 v2, s2
578; GFX7-NEXT:    flat_store_dword v[0:1], v2
579; GFX7-NEXT:    s_endpgm
580;
581; GFX10-WGP-LABEL: flat_singlethread_unordered_store:
582; GFX10-WGP:       ; %bb.0: ; %entry
583; GFX10-WGP-NEXT:    s_clause 0x1
584; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
585; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
586; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
587; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
588; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
589; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
590; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
591; GFX10-WGP-NEXT:    s_endpgm
592;
593; GFX10-CU-LABEL: flat_singlethread_unordered_store:
594; GFX10-CU:       ; %bb.0: ; %entry
595; GFX10-CU-NEXT:    s_clause 0x1
596; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
597; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
598; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
599; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
600; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
601; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
602; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
603; GFX10-CU-NEXT:    s_endpgm
604;
605; SKIP-CACHE-INV-LABEL: flat_singlethread_unordered_store:
606; SKIP-CACHE-INV:       ; %bb.0: ; %entry
607; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
608; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
609; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
610; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
611; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
612; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
613; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
614; SKIP-CACHE-INV-NEXT:    s_endpgm
615;
616; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_unordered_store:
617; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
618; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
619; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
620; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
621; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
622; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
623; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
624; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
625;
626; GFX90A-TGSPLIT-LABEL: flat_singlethread_unordered_store:
627; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
628; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
629; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
630; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
631; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
632; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
633; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
634; GFX90A-TGSPLIT-NEXT:    s_endpgm
635;
636; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_unordered_store:
637; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
638; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
639; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
640; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
641; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
642; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
643; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
644; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
645;
646; GFX940-TGSPLIT-LABEL: flat_singlethread_unordered_store:
647; GFX940-TGSPLIT:       ; %bb.0: ; %entry
648; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
649; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
650; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
651; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
652; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
653; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
654; GFX940-TGSPLIT-NEXT:    s_endpgm
655;
656; GFX11-WGP-LABEL: flat_singlethread_unordered_store:
657; GFX11-WGP:       ; %bb.0: ; %entry
658; GFX11-WGP-NEXT:    s_clause 0x1
659; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
660; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
661; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
662; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
663; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
664; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
665; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
666; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
667; GFX11-WGP-NEXT:    s_endpgm
668;
669; GFX11-CU-LABEL: flat_singlethread_unordered_store:
670; GFX11-CU:       ; %bb.0: ; %entry
671; GFX11-CU-NEXT:    s_clause 0x1
672; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
673; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
674; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
675; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
676; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
677; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
678; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
679; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
680; GFX11-CU-NEXT:    s_endpgm
681    i32 %in, i32* %out) {
682entry:
683  store atomic i32 %in, i32* %out syncscope("singlethread") unordered, align 4
684  ret void
685}
686
687define amdgpu_kernel void @flat_singlethread_monotonic_store(
688; GFX7-LABEL: flat_singlethread_monotonic_store:
689; GFX7:       ; %bb.0: ; %entry
690; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
691; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
692; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
693; GFX7-NEXT:    v_mov_b32_e32 v0, s0
694; GFX7-NEXT:    v_mov_b32_e32 v1, s1
695; GFX7-NEXT:    v_mov_b32_e32 v2, s2
696; GFX7-NEXT:    flat_store_dword v[0:1], v2
697; GFX7-NEXT:    s_endpgm
698;
699; GFX10-WGP-LABEL: flat_singlethread_monotonic_store:
700; GFX10-WGP:       ; %bb.0: ; %entry
701; GFX10-WGP-NEXT:    s_clause 0x1
702; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
703; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
704; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
705; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
706; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
707; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
708; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
709; GFX10-WGP-NEXT:    s_endpgm
710;
711; GFX10-CU-LABEL: flat_singlethread_monotonic_store:
712; GFX10-CU:       ; %bb.0: ; %entry
713; GFX10-CU-NEXT:    s_clause 0x1
714; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
715; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
716; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
717; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
718; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
719; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
720; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
721; GFX10-CU-NEXT:    s_endpgm
722;
723; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_store:
724; SKIP-CACHE-INV:       ; %bb.0: ; %entry
725; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
726; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
727; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
728; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
729; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
730; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
731; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
732; SKIP-CACHE-INV-NEXT:    s_endpgm
733;
734; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_store:
735; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
736; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
737; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
738; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
739; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
740; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
741; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
742; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
743;
744; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_store:
745; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
746; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
747; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
748; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
749; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
750; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
751; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
752; GFX90A-TGSPLIT-NEXT:    s_endpgm
753;
754; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_store:
755; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
756; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
757; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
758; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
759; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
760; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
761; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
762; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
763;
764; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_store:
765; GFX940-TGSPLIT:       ; %bb.0: ; %entry
766; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
767; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
768; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
769; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
770; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
771; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
772; GFX940-TGSPLIT-NEXT:    s_endpgm
773;
774; GFX11-WGP-LABEL: flat_singlethread_monotonic_store:
775; GFX11-WGP:       ; %bb.0: ; %entry
776; GFX11-WGP-NEXT:    s_clause 0x1
777; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
778; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
779; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
780; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
781; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
782; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
783; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
784; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
785; GFX11-WGP-NEXT:    s_endpgm
786;
787; GFX11-CU-LABEL: flat_singlethread_monotonic_store:
788; GFX11-CU:       ; %bb.0: ; %entry
789; GFX11-CU-NEXT:    s_clause 0x1
790; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
791; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
792; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
793; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
794; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
795; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
796; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
797; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
798; GFX11-CU-NEXT:    s_endpgm
799    i32 %in, i32* %out) {
800entry:
801  store atomic i32 %in, i32* %out syncscope("singlethread") monotonic, align 4
802  ret void
803}
804
805define amdgpu_kernel void @flat_singlethread_release_store(
806; GFX7-LABEL: flat_singlethread_release_store:
807; GFX7:       ; %bb.0: ; %entry
808; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
809; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
810; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
811; GFX7-NEXT:    v_mov_b32_e32 v0, s0
812; GFX7-NEXT:    v_mov_b32_e32 v1, s1
813; GFX7-NEXT:    v_mov_b32_e32 v2, s2
814; GFX7-NEXT:    flat_store_dword v[0:1], v2
815; GFX7-NEXT:    s_endpgm
816;
817; GFX10-WGP-LABEL: flat_singlethread_release_store:
818; GFX10-WGP:       ; %bb.0: ; %entry
819; GFX10-WGP-NEXT:    s_clause 0x1
820; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
821; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
822; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
823; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
824; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
825; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
826; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
827; GFX10-WGP-NEXT:    s_endpgm
828;
829; GFX10-CU-LABEL: flat_singlethread_release_store:
830; GFX10-CU:       ; %bb.0: ; %entry
831; GFX10-CU-NEXT:    s_clause 0x1
832; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
833; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
834; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
835; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
836; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
837; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
838; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
839; GFX10-CU-NEXT:    s_endpgm
840;
841; SKIP-CACHE-INV-LABEL: flat_singlethread_release_store:
842; SKIP-CACHE-INV:       ; %bb.0: ; %entry
843; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
844; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
845; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
846; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
847; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
848; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
849; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
850; SKIP-CACHE-INV-NEXT:    s_endpgm
851;
852; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_store:
853; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
854; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
855; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
856; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
857; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
858; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
859; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
860; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
861;
862; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_store:
863; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
864; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
865; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
866; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
867; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
868; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
869; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
870; GFX90A-TGSPLIT-NEXT:    s_endpgm
871;
872; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_store:
873; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
874; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
875; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
876; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
877; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
878; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
879; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
880; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
881;
882; GFX940-TGSPLIT-LABEL: flat_singlethread_release_store:
883; GFX940-TGSPLIT:       ; %bb.0: ; %entry
884; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
885; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
886; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
887; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
888; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
889; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
890; GFX940-TGSPLIT-NEXT:    s_endpgm
891;
892; GFX11-WGP-LABEL: flat_singlethread_release_store:
893; GFX11-WGP:       ; %bb.0: ; %entry
894; GFX11-WGP-NEXT:    s_clause 0x1
895; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
896; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
897; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
898; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
899; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
900; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
901; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
902; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
903; GFX11-WGP-NEXT:    s_endpgm
904;
905; GFX11-CU-LABEL: flat_singlethread_release_store:
906; GFX11-CU:       ; %bb.0: ; %entry
907; GFX11-CU-NEXT:    s_clause 0x1
908; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
909; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
910; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
911; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
912; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
913; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
914; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
915; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
916; GFX11-CU-NEXT:    s_endpgm
917    i32 %in, i32* %out) {
918entry:
919  store atomic i32 %in, i32* %out syncscope("singlethread") release, align 4
920  ret void
921}
922
923define amdgpu_kernel void @flat_singlethread_seq_cst_store(
924; GFX7-LABEL: flat_singlethread_seq_cst_store:
925; GFX7:       ; %bb.0: ; %entry
926; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
927; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
928; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
929; GFX7-NEXT:    v_mov_b32_e32 v0, s0
930; GFX7-NEXT:    v_mov_b32_e32 v1, s1
931; GFX7-NEXT:    v_mov_b32_e32 v2, s2
932; GFX7-NEXT:    flat_store_dword v[0:1], v2
933; GFX7-NEXT:    s_endpgm
934;
935; GFX10-WGP-LABEL: flat_singlethread_seq_cst_store:
936; GFX10-WGP:       ; %bb.0: ; %entry
937; GFX10-WGP-NEXT:    s_clause 0x1
938; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
939; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
940; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
941; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
942; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
943; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
944; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
945; GFX10-WGP-NEXT:    s_endpgm
946;
947; GFX10-CU-LABEL: flat_singlethread_seq_cst_store:
948; GFX10-CU:       ; %bb.0: ; %entry
949; GFX10-CU-NEXT:    s_clause 0x1
950; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
951; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
952; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
953; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
954; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
955; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
956; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
957; GFX10-CU-NEXT:    s_endpgm
958;
959; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_store:
960; SKIP-CACHE-INV:       ; %bb.0: ; %entry
961; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
962; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
963; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
964; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
965; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
966; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
967; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
968; SKIP-CACHE-INV-NEXT:    s_endpgm
969;
970; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_store:
971; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
972; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
973; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
974; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
975; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
976; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
977; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
978; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
979;
980; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_store:
981; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
982; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
983; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
984; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
985; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
986; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
987; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
988; GFX90A-TGSPLIT-NEXT:    s_endpgm
989;
990; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_store:
991; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
992; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
993; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
994; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
995; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
996; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
997; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
998; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
999;
1000; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_store:
1001; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1002; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
1003; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
1004; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1005; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1006; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1007; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1008; GFX940-TGSPLIT-NEXT:    s_endpgm
1009;
1010; GFX11-WGP-LABEL: flat_singlethread_seq_cst_store:
1011; GFX11-WGP:       ; %bb.0: ; %entry
1012; GFX11-WGP-NEXT:    s_clause 0x1
1013; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
1014; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
1015; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1016; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1017; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1018; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1019; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
1020; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1021; GFX11-WGP-NEXT:    s_endpgm
1022;
1023; GFX11-CU-LABEL: flat_singlethread_seq_cst_store:
1024; GFX11-CU:       ; %bb.0: ; %entry
1025; GFX11-CU-NEXT:    s_clause 0x1
1026; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
1027; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
1028; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1029; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1030; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1031; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1032; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1033; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1034; GFX11-CU-NEXT:    s_endpgm
1035    i32 %in, i32* %out) {
1036entry:
1037  store atomic i32 %in, i32* %out syncscope("singlethread") seq_cst, align 4
1038  ret void
1039}
1040
1041define amdgpu_kernel void @flat_singlethread_monotonic_atomicrmw(
1042; GFX7-LABEL: flat_singlethread_monotonic_atomicrmw:
1043; GFX7:       ; %bb.0: ; %entry
1044; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1045; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1046; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1047; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1048; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1049; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1050; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1051; GFX7-NEXT:    s_endpgm
1052;
1053; GFX10-WGP-LABEL: flat_singlethread_monotonic_atomicrmw:
1054; GFX10-WGP:       ; %bb.0: ; %entry
1055; GFX10-WGP-NEXT:    s_clause 0x1
1056; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1057; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1058; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1059; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1060; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1061; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1062; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1063; GFX10-WGP-NEXT:    s_endpgm
1064;
1065; GFX10-CU-LABEL: flat_singlethread_monotonic_atomicrmw:
1066; GFX10-CU:       ; %bb.0: ; %entry
1067; GFX10-CU-NEXT:    s_clause 0x1
1068; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1069; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1070; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1071; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1072; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1073; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1074; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1075; GFX10-CU-NEXT:    s_endpgm
1076;
1077; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_atomicrmw:
1078; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1079; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1080; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1081; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1082; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1083; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1084; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1085; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1086; SKIP-CACHE-INV-NEXT:    s_endpgm
1087;
1088; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_atomicrmw:
1089; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1090; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1091; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1092; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1093; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1094; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1095; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1096; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1097;
1098; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_atomicrmw:
1099; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1100; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1101; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1102; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1103; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1104; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1105; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1106; GFX90A-TGSPLIT-NEXT:    s_endpgm
1107;
1108; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_atomicrmw:
1109; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1110; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1111; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1112; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1113; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1114; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1115; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1116; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1117;
1118; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_atomicrmw:
1119; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1120; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1121; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1122; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1123; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1124; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1125; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1126; GFX940-TGSPLIT-NEXT:    s_endpgm
1127;
1128; GFX11-WGP-LABEL: flat_singlethread_monotonic_atomicrmw:
1129; GFX11-WGP:       ; %bb.0: ; %entry
1130; GFX11-WGP-NEXT:    s_clause 0x1
1131; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1132; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1133; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1134; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1135; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1136; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1137; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1138; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1139; GFX11-WGP-NEXT:    s_endpgm
1140;
1141; GFX11-CU-LABEL: flat_singlethread_monotonic_atomicrmw:
1142; GFX11-CU:       ; %bb.0: ; %entry
1143; GFX11-CU-NEXT:    s_clause 0x1
1144; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1145; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1146; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1147; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1148; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1149; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1150; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1151; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1152; GFX11-CU-NEXT:    s_endpgm
1153    i32* %out, i32 %in) {
1154entry:
1155  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") monotonic
1156  ret void
1157}
1158
1159define amdgpu_kernel void @flat_singlethread_acquire_atomicrmw(
1160; GFX7-LABEL: flat_singlethread_acquire_atomicrmw:
1161; GFX7:       ; %bb.0: ; %entry
1162; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1163; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1164; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1165; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1166; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1167; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1168; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1169; GFX7-NEXT:    s_endpgm
1170;
1171; GFX10-WGP-LABEL: flat_singlethread_acquire_atomicrmw:
1172; GFX10-WGP:       ; %bb.0: ; %entry
1173; GFX10-WGP-NEXT:    s_clause 0x1
1174; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1175; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1176; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1177; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1178; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1179; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1180; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1181; GFX10-WGP-NEXT:    s_endpgm
1182;
1183; GFX10-CU-LABEL: flat_singlethread_acquire_atomicrmw:
1184; GFX10-CU:       ; %bb.0: ; %entry
1185; GFX10-CU-NEXT:    s_clause 0x1
1186; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1187; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1188; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1189; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1190; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1191; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1192; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1193; GFX10-CU-NEXT:    s_endpgm
1194;
1195; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_atomicrmw:
1196; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1197; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1198; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1199; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1200; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1201; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1202; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1203; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1204; SKIP-CACHE-INV-NEXT:    s_endpgm
1205;
1206; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_atomicrmw:
1207; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1208; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1209; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1210; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1211; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1212; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1213; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1214; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1215;
1216; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_atomicrmw:
1217; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1218; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1219; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1220; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1221; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1222; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1223; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1224; GFX90A-TGSPLIT-NEXT:    s_endpgm
1225;
1226; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_atomicrmw:
1227; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1228; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1229; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1230; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1231; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1232; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1233; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1234; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1235;
1236; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_atomicrmw:
1237; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1238; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1239; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1240; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1241; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1242; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1243; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1244; GFX940-TGSPLIT-NEXT:    s_endpgm
1245;
1246; GFX11-WGP-LABEL: flat_singlethread_acquire_atomicrmw:
1247; GFX11-WGP:       ; %bb.0: ; %entry
1248; GFX11-WGP-NEXT:    s_clause 0x1
1249; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1250; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1251; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1252; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1253; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1254; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1255; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1256; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1257; GFX11-WGP-NEXT:    s_endpgm
1258;
1259; GFX11-CU-LABEL: flat_singlethread_acquire_atomicrmw:
1260; GFX11-CU:       ; %bb.0: ; %entry
1261; GFX11-CU-NEXT:    s_clause 0x1
1262; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1263; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1264; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1265; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1266; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1267; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1268; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1269; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1270; GFX11-CU-NEXT:    s_endpgm
1271    i32* %out, i32 %in) {
1272entry:
1273  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acquire
1274  ret void
1275}
1276
1277define amdgpu_kernel void @flat_singlethread_release_atomicrmw(
1278; GFX7-LABEL: flat_singlethread_release_atomicrmw:
1279; GFX7:       ; %bb.0: ; %entry
1280; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1281; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1282; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1283; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1284; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1285; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1286; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1287; GFX7-NEXT:    s_endpgm
1288;
1289; GFX10-WGP-LABEL: flat_singlethread_release_atomicrmw:
1290; GFX10-WGP:       ; %bb.0: ; %entry
1291; GFX10-WGP-NEXT:    s_clause 0x1
1292; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1293; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1294; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1295; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1296; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1297; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1298; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1299; GFX10-WGP-NEXT:    s_endpgm
1300;
1301; GFX10-CU-LABEL: flat_singlethread_release_atomicrmw:
1302; GFX10-CU:       ; %bb.0: ; %entry
1303; GFX10-CU-NEXT:    s_clause 0x1
1304; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1305; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1306; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1307; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1308; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1309; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1310; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1311; GFX10-CU-NEXT:    s_endpgm
1312;
1313; SKIP-CACHE-INV-LABEL: flat_singlethread_release_atomicrmw:
1314; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1315; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1316; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1317; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1318; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1319; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1320; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1321; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1322; SKIP-CACHE-INV-NEXT:    s_endpgm
1323;
1324; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_atomicrmw:
1325; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1326; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1327; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1328; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1329; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1330; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1331; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1332; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1333;
1334; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_atomicrmw:
1335; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1336; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1337; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1338; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1339; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1340; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1341; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1342; GFX90A-TGSPLIT-NEXT:    s_endpgm
1343;
1344; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_atomicrmw:
1345; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1346; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1347; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1348; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1349; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1350; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1351; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1352; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1353;
1354; GFX940-TGSPLIT-LABEL: flat_singlethread_release_atomicrmw:
1355; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1356; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1357; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1358; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1359; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1360; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1361; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1362; GFX940-TGSPLIT-NEXT:    s_endpgm
1363;
1364; GFX11-WGP-LABEL: flat_singlethread_release_atomicrmw:
1365; GFX11-WGP:       ; %bb.0: ; %entry
1366; GFX11-WGP-NEXT:    s_clause 0x1
1367; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1368; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1369; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1370; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1371; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1372; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1373; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1374; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1375; GFX11-WGP-NEXT:    s_endpgm
1376;
1377; GFX11-CU-LABEL: flat_singlethread_release_atomicrmw:
1378; GFX11-CU:       ; %bb.0: ; %entry
1379; GFX11-CU-NEXT:    s_clause 0x1
1380; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1381; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1382; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1383; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1384; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1385; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1386; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1387; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1388; GFX11-CU-NEXT:    s_endpgm
1389    i32* %out, i32 %in) {
1390entry:
1391  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") release
1392  ret void
1393}
1394
1395define amdgpu_kernel void @flat_singlethread_acq_rel_atomicrmw(
1396; GFX7-LABEL: flat_singlethread_acq_rel_atomicrmw:
1397; GFX7:       ; %bb.0: ; %entry
1398; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1399; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1400; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1401; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1402; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1403; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1404; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1405; GFX7-NEXT:    s_endpgm
1406;
1407; GFX10-WGP-LABEL: flat_singlethread_acq_rel_atomicrmw:
1408; GFX10-WGP:       ; %bb.0: ; %entry
1409; GFX10-WGP-NEXT:    s_clause 0x1
1410; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1411; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1412; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1413; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1414; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1415; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1416; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1417; GFX10-WGP-NEXT:    s_endpgm
1418;
1419; GFX10-CU-LABEL: flat_singlethread_acq_rel_atomicrmw:
1420; GFX10-CU:       ; %bb.0: ; %entry
1421; GFX10-CU-NEXT:    s_clause 0x1
1422; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1423; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1424; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1425; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1426; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1427; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1428; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1429; GFX10-CU-NEXT:    s_endpgm
1430;
1431; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_atomicrmw:
1432; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1433; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1434; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1435; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1436; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1437; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1438; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1439; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1440; SKIP-CACHE-INV-NEXT:    s_endpgm
1441;
1442; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_atomicrmw:
1443; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1444; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1445; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1446; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1447; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1448; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1449; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1450; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1451;
1452; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_atomicrmw:
1453; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1454; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1455; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1456; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1457; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1458; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1459; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1460; GFX90A-TGSPLIT-NEXT:    s_endpgm
1461;
1462; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_atomicrmw:
1463; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1464; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1465; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1466; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1467; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1468; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1469; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1470; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1471;
1472; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_atomicrmw:
1473; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1474; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1475; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1476; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1477; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1478; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1479; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1480; GFX940-TGSPLIT-NEXT:    s_endpgm
1481;
1482; GFX11-WGP-LABEL: flat_singlethread_acq_rel_atomicrmw:
1483; GFX11-WGP:       ; %bb.0: ; %entry
1484; GFX11-WGP-NEXT:    s_clause 0x1
1485; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1486; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1487; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1488; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1489; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1490; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1491; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1492; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1493; GFX11-WGP-NEXT:    s_endpgm
1494;
1495; GFX11-CU-LABEL: flat_singlethread_acq_rel_atomicrmw:
1496; GFX11-CU:       ; %bb.0: ; %entry
1497; GFX11-CU-NEXT:    s_clause 0x1
1498; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1499; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1500; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1501; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1502; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1503; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1504; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1505; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1506; GFX11-CU-NEXT:    s_endpgm
1507    i32* %out, i32 %in) {
1508entry:
1509  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acq_rel
1510  ret void
1511}
1512
1513define amdgpu_kernel void @flat_singlethread_seq_cst_atomicrmw(
1514; GFX7-LABEL: flat_singlethread_seq_cst_atomicrmw:
1515; GFX7:       ; %bb.0: ; %entry
1516; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1517; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1518; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1519; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1520; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1521; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1522; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1523; GFX7-NEXT:    s_endpgm
1524;
1525; GFX10-WGP-LABEL: flat_singlethread_seq_cst_atomicrmw:
1526; GFX10-WGP:       ; %bb.0: ; %entry
1527; GFX10-WGP-NEXT:    s_clause 0x1
1528; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1529; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1530; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1531; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1532; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1533; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1534; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1535; GFX10-WGP-NEXT:    s_endpgm
1536;
1537; GFX10-CU-LABEL: flat_singlethread_seq_cst_atomicrmw:
1538; GFX10-CU:       ; %bb.0: ; %entry
1539; GFX10-CU-NEXT:    s_clause 0x1
1540; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1541; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1542; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1543; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1544; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1545; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1546; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1547; GFX10-CU-NEXT:    s_endpgm
1548;
1549; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_atomicrmw:
1550; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1551; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1552; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1553; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1554; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1555; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1556; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1557; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1558; SKIP-CACHE-INV-NEXT:    s_endpgm
1559;
1560; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_atomicrmw:
1561; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1562; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1563; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1564; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1565; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1566; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1567; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1568; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1569;
1570; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_atomicrmw:
1571; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1572; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1573; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1574; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1575; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1576; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1577; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1578; GFX90A-TGSPLIT-NEXT:    s_endpgm
1579;
1580; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_atomicrmw:
1581; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1582; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1583; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1584; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1585; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1586; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1587; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1588; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1589;
1590; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_atomicrmw:
1591; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1592; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1593; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1594; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1595; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1596; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1597; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1598; GFX940-TGSPLIT-NEXT:    s_endpgm
1599;
1600; GFX11-WGP-LABEL: flat_singlethread_seq_cst_atomicrmw:
1601; GFX11-WGP:       ; %bb.0: ; %entry
1602; GFX11-WGP-NEXT:    s_clause 0x1
1603; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1604; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1605; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1606; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1607; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1608; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1609; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1610; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1611; GFX11-WGP-NEXT:    s_endpgm
1612;
1613; GFX11-CU-LABEL: flat_singlethread_seq_cst_atomicrmw:
1614; GFX11-CU:       ; %bb.0: ; %entry
1615; GFX11-CU-NEXT:    s_clause 0x1
1616; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1617; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1618; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1619; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1620; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1621; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1622; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1623; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1624; GFX11-CU-NEXT:    s_endpgm
1625    i32* %out, i32 %in) {
1626entry:
1627  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") seq_cst
1628  ret void
1629}
1630
1631define amdgpu_kernel void @flat_singlethread_acquire_ret_atomicrmw(
1632; GFX7-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1633; GFX7:       ; %bb.0: ; %entry
1634; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1635; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1636; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1637; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1638; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1639; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1640; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1641; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1642; GFX7-NEXT:    flat_store_dword v[0:1], v2
1643; GFX7-NEXT:    s_endpgm
1644;
1645; GFX10-WGP-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1646; GFX10-WGP:       ; %bb.0: ; %entry
1647; GFX10-WGP-NEXT:    s_clause 0x1
1648; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1649; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1650; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1651; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1652; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1653; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1654; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1655; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1656; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1657; GFX10-WGP-NEXT:    s_endpgm
1658;
1659; GFX10-CU-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1660; GFX10-CU:       ; %bb.0: ; %entry
1661; GFX10-CU-NEXT:    s_clause 0x1
1662; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1663; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1664; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1665; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1666; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1667; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1668; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1669; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1670; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1671; GFX10-CU-NEXT:    s_endpgm
1672;
1673; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1674; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1675; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1676; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1677; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1678; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1679; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1680; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1681; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1682; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1683; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1684; SKIP-CACHE-INV-NEXT:    s_endpgm
1685;
1686; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1687; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1688; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1689; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1690; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1691; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1692; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1693; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1694; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1695; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1696; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1697;
1698; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1699; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1700; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1701; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1702; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1703; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1704; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1705; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1706; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1707; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1708; GFX90A-TGSPLIT-NEXT:    s_endpgm
1709;
1710; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1711; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1712; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1713; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1714; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1715; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1716; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1717; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1718; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1719; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1720; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1721;
1722; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1723; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1724; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1725; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1726; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1727; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1728; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1729; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1730; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1731; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1732; GFX940-TGSPLIT-NEXT:    s_endpgm
1733;
1734; GFX11-WGP-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1735; GFX11-WGP:       ; %bb.0: ; %entry
1736; GFX11-WGP-NEXT:    s_clause 0x1
1737; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1738; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1739; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1740; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1741; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1742; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1743; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1744; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1745; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
1746; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1747; GFX11-WGP-NEXT:    s_endpgm
1748;
1749; GFX11-CU-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1750; GFX11-CU:       ; %bb.0: ; %entry
1751; GFX11-CU-NEXT:    s_clause 0x1
1752; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1753; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1754; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1755; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1756; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1757; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1758; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1759; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1760; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1761; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1762; GFX11-CU-NEXT:    s_endpgm
1763    i32* %out, i32 %in) {
1764entry:
1765  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acquire
1766  store i32 %val, i32* %out, align 4
1767  ret void
1768}
1769
1770define amdgpu_kernel void @flat_singlethread_acq_rel_ret_atomicrmw(
1771; GFX7-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1772; GFX7:       ; %bb.0: ; %entry
1773; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1774; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1775; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1776; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1777; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1778; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1779; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1780; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1781; GFX7-NEXT:    flat_store_dword v[0:1], v2
1782; GFX7-NEXT:    s_endpgm
1783;
1784; GFX10-WGP-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1785; GFX10-WGP:       ; %bb.0: ; %entry
1786; GFX10-WGP-NEXT:    s_clause 0x1
1787; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1788; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1789; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1790; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1791; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1792; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1793; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1794; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1795; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1796; GFX10-WGP-NEXT:    s_endpgm
1797;
1798; GFX10-CU-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1799; GFX10-CU:       ; %bb.0: ; %entry
1800; GFX10-CU-NEXT:    s_clause 0x1
1801; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1802; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1803; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1804; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1805; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1806; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1807; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1808; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1809; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1810; GFX10-CU-NEXT:    s_endpgm
1811;
1812; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1813; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1814; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1815; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1816; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1817; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1818; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1819; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1820; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1821; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1822; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1823; SKIP-CACHE-INV-NEXT:    s_endpgm
1824;
1825; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1826; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1827; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1828; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1829; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1830; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1831; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1832; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1833; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1834; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1835; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1836;
1837; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1838; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1839; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1840; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1841; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1842; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1843; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1844; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1845; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1846; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1847; GFX90A-TGSPLIT-NEXT:    s_endpgm
1848;
1849; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1850; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1851; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1852; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1853; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1854; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1855; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1856; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1857; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1858; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1859; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1860;
1861; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1862; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1863; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1864; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1865; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1866; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1867; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1868; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1869; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1870; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1871; GFX940-TGSPLIT-NEXT:    s_endpgm
1872;
1873; GFX11-WGP-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1874; GFX11-WGP:       ; %bb.0: ; %entry
1875; GFX11-WGP-NEXT:    s_clause 0x1
1876; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1877; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1878; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1879; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1880; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1881; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1882; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1883; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1884; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
1885; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1886; GFX11-WGP-NEXT:    s_endpgm
1887;
1888; GFX11-CU-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1889; GFX11-CU:       ; %bb.0: ; %entry
1890; GFX11-CU-NEXT:    s_clause 0x1
1891; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1892; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1893; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1894; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1895; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1896; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1897; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1898; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1899; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1900; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1901; GFX11-CU-NEXT:    s_endpgm
1902    i32* %out, i32 %in) {
1903entry:
1904  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acq_rel
1905  store i32 %val, i32* %out, align 4
1906  ret void
1907}
1908
1909define amdgpu_kernel void @flat_singlethread_seq_cst_ret_atomicrmw(
1910; GFX7-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1911; GFX7:       ; %bb.0: ; %entry
1912; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1913; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1914; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1915; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1916; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1917; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1918; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1919; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1920; GFX7-NEXT:    flat_store_dword v[0:1], v2
1921; GFX7-NEXT:    s_endpgm
1922;
1923; GFX10-WGP-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1924; GFX10-WGP:       ; %bb.0: ; %entry
1925; GFX10-WGP-NEXT:    s_clause 0x1
1926; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1927; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1928; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1929; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1930; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1931; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1932; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1933; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1934; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1935; GFX10-WGP-NEXT:    s_endpgm
1936;
1937; GFX10-CU-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1938; GFX10-CU:       ; %bb.0: ; %entry
1939; GFX10-CU-NEXT:    s_clause 0x1
1940; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1941; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1942; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1943; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1944; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1945; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1946; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1947; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1948; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1949; GFX10-CU-NEXT:    s_endpgm
1950;
1951; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1952; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1953; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1954; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1955; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1956; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1957; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1958; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1959; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1960; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1961; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1962; SKIP-CACHE-INV-NEXT:    s_endpgm
1963;
1964; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1965; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1966; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1967; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1968; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1969; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1970; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1971; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1972; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1973; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1974; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1975;
1976; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1977; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1978; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1979; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1980; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1981; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1982; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1983; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1984; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1985; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1986; GFX90A-TGSPLIT-NEXT:    s_endpgm
1987;
1988; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1989; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1990; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1991; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1992; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1993; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1994; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1995; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1996; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1997; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1998; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1999;
2000; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
2001; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2002; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2003; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2004; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2005; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2006; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2007; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
2008; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2009; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2010; GFX940-TGSPLIT-NEXT:    s_endpgm
2011;
2012; GFX11-WGP-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
2013; GFX11-WGP:       ; %bb.0: ; %entry
2014; GFX11-WGP-NEXT:    s_clause 0x1
2015; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2016; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
2017; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2018; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2019; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2020; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2021; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2022; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2023; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
2024; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2025; GFX11-WGP-NEXT:    s_endpgm
2026;
2027; GFX11-CU-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
2028; GFX11-CU:       ; %bb.0: ; %entry
2029; GFX11-CU-NEXT:    s_clause 0x1
2030; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2031; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
2032; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2033; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2034; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2035; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2036; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2037; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2038; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
2039; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2040; GFX11-CU-NEXT:    s_endpgm
2041    i32* %out, i32 %in) {
2042entry:
2043  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") seq_cst
2044  store i32 %val, i32* %out, align 4
2045  ret void
2046}
2047
2048define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_cmpxchg(
2049; GFX7-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
2050; GFX7:       ; %bb.0: ; %entry
2051; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2052; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2053; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2054; GFX7-NEXT:    s_add_u32 s0, s0, 16
2055; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2056; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2057; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2058; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2059; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2060; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2061; GFX7-NEXT:    s_endpgm
2062;
2063; GFX10-WGP-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
2064; GFX10-WGP:       ; %bb.0: ; %entry
2065; GFX10-WGP-NEXT:    s_clause 0x1
2066; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2067; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2068; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2069; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2070; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2071; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2072; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2073; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2074; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2075; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2076; GFX10-WGP-NEXT:    s_endpgm
2077;
2078; GFX10-CU-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
2079; GFX10-CU:       ; %bb.0: ; %entry
2080; GFX10-CU-NEXT:    s_clause 0x1
2081; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2082; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2083; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2084; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2085; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2086; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2087; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2088; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2089; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2090; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2091; GFX10-CU-NEXT:    s_endpgm
2092;
2093; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
2094; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2095; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2096; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2097; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2098; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2099; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2100; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2101; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2102; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2103; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2104; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2105; SKIP-CACHE-INV-NEXT:    s_endpgm
2106;
2107; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
2108; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2109; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2110; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2111; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2112; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2113; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2114; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2115; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2116;
2117; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
2118; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2119; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2120; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2121; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2122; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2123; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2124; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2125; GFX90A-TGSPLIT-NEXT:    s_endpgm
2126;
2127; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
2128; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2129; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2130; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2131; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2132; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2133; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2134; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2135; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2136;
2137; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
2138; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2139; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2140; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2141; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2142; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2143; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2144; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2145; GFX940-TGSPLIT-NEXT:    s_endpgm
2146;
2147; GFX11-WGP-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
2148; GFX11-WGP:       ; %bb.0: ; %entry
2149; GFX11-WGP-NEXT:    s_clause 0x1
2150; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2151; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2152; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2153; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2154; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2155; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2156; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2157; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2158; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2159; GFX11-WGP-NEXT:    s_endpgm
2160;
2161; GFX11-CU-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
2162; GFX11-CU:       ; %bb.0: ; %entry
2163; GFX11-CU-NEXT:    s_clause 0x1
2164; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2165; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2166; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2167; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2168; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2169; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2170; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2171; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2172; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2173; GFX11-CU-NEXT:    s_endpgm
2174    i32* %out, i32 %in, i32 %old) {
2175entry:
2176  %gep = getelementptr i32, i32* %out, i32 4
2177  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
2178  ret void
2179}
2180
2181define amdgpu_kernel void @flat_singlethread_acquire_monotonic_cmpxchg(
2182; GFX7-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
2183; GFX7:       ; %bb.0: ; %entry
2184; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2185; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2186; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2187; GFX7-NEXT:    s_add_u32 s0, s0, 16
2188; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2189; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2190; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2191; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2192; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2193; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2194; GFX7-NEXT:    s_endpgm
2195;
2196; GFX10-WGP-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
2197; GFX10-WGP:       ; %bb.0: ; %entry
2198; GFX10-WGP-NEXT:    s_clause 0x1
2199; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2200; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2201; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2202; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2203; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2204; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2205; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2206; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2207; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2208; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2209; GFX10-WGP-NEXT:    s_endpgm
2210;
2211; GFX10-CU-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
2212; GFX10-CU:       ; %bb.0: ; %entry
2213; GFX10-CU-NEXT:    s_clause 0x1
2214; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2215; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2216; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2217; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2218; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2219; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2220; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2221; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2222; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2223; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2224; GFX10-CU-NEXT:    s_endpgm
2225;
2226; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
2227; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2228; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2229; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2230; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2231; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2232; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2233; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2234; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2235; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2236; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2237; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2238; SKIP-CACHE-INV-NEXT:    s_endpgm
2239;
2240; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
2241; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2242; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2243; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2244; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2245; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2246; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2247; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2248; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2249;
2250; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
2251; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2252; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2253; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2254; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2255; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2256; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2257; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2258; GFX90A-TGSPLIT-NEXT:    s_endpgm
2259;
2260; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
2261; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2262; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2263; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2264; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2265; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2266; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2267; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2268; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2269;
2270; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
2271; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2272; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2273; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2274; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2275; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2276; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2277; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2278; GFX940-TGSPLIT-NEXT:    s_endpgm
2279;
2280; GFX11-WGP-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
2281; GFX11-WGP:       ; %bb.0: ; %entry
2282; GFX11-WGP-NEXT:    s_clause 0x1
2283; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2284; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2285; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2286; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2287; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2288; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2289; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2290; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2291; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2292; GFX11-WGP-NEXT:    s_endpgm
2293;
2294; GFX11-CU-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
2295; GFX11-CU:       ; %bb.0: ; %entry
2296; GFX11-CU-NEXT:    s_clause 0x1
2297; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2298; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2299; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2300; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2301; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2302; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2303; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2304; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2305; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2306; GFX11-CU-NEXT:    s_endpgm
2307    i32* %out, i32 %in, i32 %old) {
2308entry:
2309  %gep = getelementptr i32, i32* %out, i32 4
2310  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
2311  ret void
2312}
2313
2314define amdgpu_kernel void @flat_singlethread_release_monotonic_cmpxchg(
2315; GFX7-LABEL: flat_singlethread_release_monotonic_cmpxchg:
2316; GFX7:       ; %bb.0: ; %entry
2317; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2318; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2319; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2320; GFX7-NEXT:    s_add_u32 s0, s0, 16
2321; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2322; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2323; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2324; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2325; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2326; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2327; GFX7-NEXT:    s_endpgm
2328;
2329; GFX10-WGP-LABEL: flat_singlethread_release_monotonic_cmpxchg:
2330; GFX10-WGP:       ; %bb.0: ; %entry
2331; GFX10-WGP-NEXT:    s_clause 0x1
2332; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2333; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2334; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2335; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2336; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2337; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2338; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2339; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2340; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2341; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2342; GFX10-WGP-NEXT:    s_endpgm
2343;
2344; GFX10-CU-LABEL: flat_singlethread_release_monotonic_cmpxchg:
2345; GFX10-CU:       ; %bb.0: ; %entry
2346; GFX10-CU-NEXT:    s_clause 0x1
2347; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2348; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2349; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2350; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2351; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2352; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2353; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2354; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2355; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2356; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2357; GFX10-CU-NEXT:    s_endpgm
2358;
2359; SKIP-CACHE-INV-LABEL: flat_singlethread_release_monotonic_cmpxchg:
2360; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2361; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2362; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2363; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2364; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2365; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2366; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2367; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2368; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2369; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2370; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2371; SKIP-CACHE-INV-NEXT:    s_endpgm
2372;
2373; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_monotonic_cmpxchg:
2374; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2375; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2376; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2377; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2378; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2379; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2380; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2381; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2382;
2383; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_monotonic_cmpxchg:
2384; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2385; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2386; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2387; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2388; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2389; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2390; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2391; GFX90A-TGSPLIT-NEXT:    s_endpgm
2392;
2393; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_monotonic_cmpxchg:
2394; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2395; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2396; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2397; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2398; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2399; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2400; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2401; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2402;
2403; GFX940-TGSPLIT-LABEL: flat_singlethread_release_monotonic_cmpxchg:
2404; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2405; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2406; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2407; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2408; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2409; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2410; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2411; GFX940-TGSPLIT-NEXT:    s_endpgm
2412;
2413; GFX11-WGP-LABEL: flat_singlethread_release_monotonic_cmpxchg:
2414; GFX11-WGP:       ; %bb.0: ; %entry
2415; GFX11-WGP-NEXT:    s_clause 0x1
2416; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2417; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2418; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2419; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2420; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2421; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2422; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2423; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2424; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2425; GFX11-WGP-NEXT:    s_endpgm
2426;
2427; GFX11-CU-LABEL: flat_singlethread_release_monotonic_cmpxchg:
2428; GFX11-CU:       ; %bb.0: ; %entry
2429; GFX11-CU-NEXT:    s_clause 0x1
2430; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2431; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2432; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2433; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2434; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2435; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2436; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2437; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2438; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2439; GFX11-CU-NEXT:    s_endpgm
2440    i32* %out, i32 %in, i32 %old) {
2441entry:
2442  %gep = getelementptr i32, i32* %out, i32 4
2443  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
2444  ret void
2445}
2446
2447define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_cmpxchg(
2448; GFX7-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2449; GFX7:       ; %bb.0: ; %entry
2450; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2451; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2452; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2453; GFX7-NEXT:    s_add_u32 s0, s0, 16
2454; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2455; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2456; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2457; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2458; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2459; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2460; GFX7-NEXT:    s_endpgm
2461;
2462; GFX10-WGP-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2463; GFX10-WGP:       ; %bb.0: ; %entry
2464; GFX10-WGP-NEXT:    s_clause 0x1
2465; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2466; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2467; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2468; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2469; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2470; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2471; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2472; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2473; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2474; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2475; GFX10-WGP-NEXT:    s_endpgm
2476;
2477; GFX10-CU-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2478; GFX10-CU:       ; %bb.0: ; %entry
2479; GFX10-CU-NEXT:    s_clause 0x1
2480; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2481; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2482; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2483; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2484; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2485; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2486; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2487; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2488; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2489; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2490; GFX10-CU-NEXT:    s_endpgm
2491;
2492; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2493; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2494; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2495; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2496; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2497; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2498; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2499; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2500; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2501; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2502; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2503; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2504; SKIP-CACHE-INV-NEXT:    s_endpgm
2505;
2506; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2507; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2508; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2509; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2510; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2511; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2512; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2513; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2514; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2515;
2516; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2517; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2518; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2519; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2520; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2521; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2522; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2523; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2524; GFX90A-TGSPLIT-NEXT:    s_endpgm
2525;
2526; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2527; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2528; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2529; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2530; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2531; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2532; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2533; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2534; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2535;
2536; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2537; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2538; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2539; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2540; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2541; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2542; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2543; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2544; GFX940-TGSPLIT-NEXT:    s_endpgm
2545;
2546; GFX11-WGP-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2547; GFX11-WGP:       ; %bb.0: ; %entry
2548; GFX11-WGP-NEXT:    s_clause 0x1
2549; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2550; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2551; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2552; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2553; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2554; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2555; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2556; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2557; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2558; GFX11-WGP-NEXT:    s_endpgm
2559;
2560; GFX11-CU-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2561; GFX11-CU:       ; %bb.0: ; %entry
2562; GFX11-CU-NEXT:    s_clause 0x1
2563; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2564; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2565; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2566; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2567; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2568; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2569; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2570; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2571; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2572; GFX11-CU-NEXT:    s_endpgm
2573    i32* %out, i32 %in, i32 %old) {
2574entry:
2575  %gep = getelementptr i32, i32* %out, i32 4
2576  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
2577  ret void
2578}
2579
2580define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_cmpxchg(
2581; GFX7-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2582; GFX7:       ; %bb.0: ; %entry
2583; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2584; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2585; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2586; GFX7-NEXT:    s_add_u32 s0, s0, 16
2587; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2588; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2589; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2590; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2591; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2592; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2593; GFX7-NEXT:    s_endpgm
2594;
2595; GFX10-WGP-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2596; GFX10-WGP:       ; %bb.0: ; %entry
2597; GFX10-WGP-NEXT:    s_clause 0x1
2598; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2599; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2600; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2601; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2602; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2603; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2604; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2605; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2606; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2607; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2608; GFX10-WGP-NEXT:    s_endpgm
2609;
2610; GFX10-CU-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2611; GFX10-CU:       ; %bb.0: ; %entry
2612; GFX10-CU-NEXT:    s_clause 0x1
2613; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2614; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2615; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2616; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2617; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2618; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2619; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2620; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2621; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2622; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2623; GFX10-CU-NEXT:    s_endpgm
2624;
2625; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2626; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2627; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2628; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2629; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2630; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2631; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2632; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2633; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2634; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2635; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2636; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2637; SKIP-CACHE-INV-NEXT:    s_endpgm
2638;
2639; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2640; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2641; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2642; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2643; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2644; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2645; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2646; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2647; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2648;
2649; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2650; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2651; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2652; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2653; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2654; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2655; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2656; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2657; GFX90A-TGSPLIT-NEXT:    s_endpgm
2658;
2659; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2660; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2661; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2662; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2663; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2664; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2665; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2666; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2667; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2668;
2669; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2670; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2671; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2672; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2673; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2674; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2675; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2676; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2677; GFX940-TGSPLIT-NEXT:    s_endpgm
2678;
2679; GFX11-WGP-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2680; GFX11-WGP:       ; %bb.0: ; %entry
2681; GFX11-WGP-NEXT:    s_clause 0x1
2682; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2683; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2684; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2685; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2686; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2687; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2688; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2689; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2690; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2691; GFX11-WGP-NEXT:    s_endpgm
2692;
2693; GFX11-CU-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2694; GFX11-CU:       ; %bb.0: ; %entry
2695; GFX11-CU-NEXT:    s_clause 0x1
2696; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2697; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2698; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2699; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2700; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2701; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2702; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2703; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2704; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2705; GFX11-CU-NEXT:    s_endpgm
2706    i32* %out, i32 %in, i32 %old) {
2707entry:
2708  %gep = getelementptr i32, i32* %out, i32 4
2709  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
2710  ret void
2711}
2712
2713define amdgpu_kernel void @flat_singlethread_monotonic_acquire_cmpxchg(
2714; GFX7-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2715; GFX7:       ; %bb.0: ; %entry
2716; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2717; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2718; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2719; GFX7-NEXT:    s_add_u32 s0, s0, 16
2720; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2721; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2722; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2723; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2724; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2725; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2726; GFX7-NEXT:    s_endpgm
2727;
2728; GFX10-WGP-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2729; GFX10-WGP:       ; %bb.0: ; %entry
2730; GFX10-WGP-NEXT:    s_clause 0x1
2731; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2732; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2733; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2734; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2735; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2736; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2737; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2738; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2739; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2740; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2741; GFX10-WGP-NEXT:    s_endpgm
2742;
2743; GFX10-CU-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2744; GFX10-CU:       ; %bb.0: ; %entry
2745; GFX10-CU-NEXT:    s_clause 0x1
2746; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2747; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2748; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2749; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2750; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2751; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2752; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2753; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2754; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2755; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2756; GFX10-CU-NEXT:    s_endpgm
2757;
2758; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2759; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2760; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2761; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2762; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2763; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2764; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2765; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2766; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2767; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2768; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2769; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2770; SKIP-CACHE-INV-NEXT:    s_endpgm
2771;
2772; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2773; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2774; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2775; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2776; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2777; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2778; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2779; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2780; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2781;
2782; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2783; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2784; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2785; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2786; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2787; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2788; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2789; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2790; GFX90A-TGSPLIT-NEXT:    s_endpgm
2791;
2792; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2793; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2794; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2795; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2796; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2797; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2798; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2799; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2800; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2801;
2802; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2803; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2804; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2805; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2806; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2807; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2808; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2809; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2810; GFX940-TGSPLIT-NEXT:    s_endpgm
2811;
2812; GFX11-WGP-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2813; GFX11-WGP:       ; %bb.0: ; %entry
2814; GFX11-WGP-NEXT:    s_clause 0x1
2815; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2816; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2817; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2818; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2819; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2820; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2821; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2822; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2823; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2824; GFX11-WGP-NEXT:    s_endpgm
2825;
2826; GFX11-CU-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2827; GFX11-CU:       ; %bb.0: ; %entry
2828; GFX11-CU-NEXT:    s_clause 0x1
2829; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2830; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2831; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2832; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2833; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2834; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2835; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2836; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2837; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2838; GFX11-CU-NEXT:    s_endpgm
2839    i32* %out, i32 %in, i32 %old) {
2840entry:
2841  %gep = getelementptr i32, i32* %out, i32 4
2842  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic acquire
2843  ret void
2844}
2845
2846define amdgpu_kernel void @flat_singlethread_acquire_acquire_cmpxchg(
2847; GFX7-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2848; GFX7:       ; %bb.0: ; %entry
2849; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2850; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2851; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2852; GFX7-NEXT:    s_add_u32 s0, s0, 16
2853; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2854; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2855; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2856; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2857; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2858; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2859; GFX7-NEXT:    s_endpgm
2860;
2861; GFX10-WGP-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2862; GFX10-WGP:       ; %bb.0: ; %entry
2863; GFX10-WGP-NEXT:    s_clause 0x1
2864; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2865; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2866; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2867; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2868; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2869; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2870; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2871; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2872; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2873; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2874; GFX10-WGP-NEXT:    s_endpgm
2875;
2876; GFX10-CU-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2877; GFX10-CU:       ; %bb.0: ; %entry
2878; GFX10-CU-NEXT:    s_clause 0x1
2879; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2880; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2881; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2882; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2883; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2884; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2885; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2886; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2887; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2888; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2889; GFX10-CU-NEXT:    s_endpgm
2890;
2891; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2892; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2893; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2894; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2895; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2896; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2897; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2898; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2899; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2900; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2901; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2902; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2903; SKIP-CACHE-INV-NEXT:    s_endpgm
2904;
2905; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2906; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2907; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2908; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2909; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2910; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2911; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2912; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2913; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2914;
2915; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2916; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2917; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2918; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2919; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2920; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2921; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2922; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2923; GFX90A-TGSPLIT-NEXT:    s_endpgm
2924;
2925; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2926; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2927; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2928; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2929; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2930; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2931; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2932; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2933; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2934;
2935; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2936; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2937; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2938; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2939; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2940; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2941; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2942; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2943; GFX940-TGSPLIT-NEXT:    s_endpgm
2944;
2945; GFX11-WGP-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2946; GFX11-WGP:       ; %bb.0: ; %entry
2947; GFX11-WGP-NEXT:    s_clause 0x1
2948; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2949; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2950; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2951; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2952; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2953; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2954; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2955; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2956; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2957; GFX11-WGP-NEXT:    s_endpgm
2958;
2959; GFX11-CU-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2960; GFX11-CU:       ; %bb.0: ; %entry
2961; GFX11-CU-NEXT:    s_clause 0x1
2962; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2963; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2964; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2965; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2966; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2967; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2968; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2969; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2970; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2971; GFX11-CU-NEXT:    s_endpgm
2972    i32* %out, i32 %in, i32 %old) {
2973entry:
2974  %gep = getelementptr i32, i32* %out, i32 4
2975  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
2976  ret void
2977}
2978
2979define amdgpu_kernel void @flat_singlethread_release_acquire_cmpxchg(
2980; GFX7-LABEL: flat_singlethread_release_acquire_cmpxchg:
2981; GFX7:       ; %bb.0: ; %entry
2982; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2983; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2984; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2985; GFX7-NEXT:    s_add_u32 s0, s0, 16
2986; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2987; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2988; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2989; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2990; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2991; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2992; GFX7-NEXT:    s_endpgm
2993;
2994; GFX10-WGP-LABEL: flat_singlethread_release_acquire_cmpxchg:
2995; GFX10-WGP:       ; %bb.0: ; %entry
2996; GFX10-WGP-NEXT:    s_clause 0x1
2997; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2998; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2999; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3000; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3001; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3002; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3003; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3004; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3005; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3006; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3007; GFX10-WGP-NEXT:    s_endpgm
3008;
3009; GFX10-CU-LABEL: flat_singlethread_release_acquire_cmpxchg:
3010; GFX10-CU:       ; %bb.0: ; %entry
3011; GFX10-CU-NEXT:    s_clause 0x1
3012; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3013; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3014; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3015; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3016; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3017; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3018; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3019; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3020; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3021; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3022; GFX10-CU-NEXT:    s_endpgm
3023;
3024; SKIP-CACHE-INV-LABEL: flat_singlethread_release_acquire_cmpxchg:
3025; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3026; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3027; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3028; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3029; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3030; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3031; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3032; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3033; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3034; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3035; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3036; SKIP-CACHE-INV-NEXT:    s_endpgm
3037;
3038; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_acquire_cmpxchg:
3039; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3040; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3041; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3042; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3043; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3044; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3045; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3046; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3047;
3048; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_acquire_cmpxchg:
3049; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3050; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3051; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3052; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3053; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3054; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3055; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3056; GFX90A-TGSPLIT-NEXT:    s_endpgm
3057;
3058; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_acquire_cmpxchg:
3059; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3060; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3061; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3062; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3063; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3064; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3065; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3066; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3067;
3068; GFX940-TGSPLIT-LABEL: flat_singlethread_release_acquire_cmpxchg:
3069; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3070; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3071; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3072; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3073; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3074; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3075; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3076; GFX940-TGSPLIT-NEXT:    s_endpgm
3077;
3078; GFX11-WGP-LABEL: flat_singlethread_release_acquire_cmpxchg:
3079; GFX11-WGP:       ; %bb.0: ; %entry
3080; GFX11-WGP-NEXT:    s_clause 0x1
3081; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3082; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3083; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3084; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3085; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3086; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3087; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3088; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3089; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3090; GFX11-WGP-NEXT:    s_endpgm
3091;
3092; GFX11-CU-LABEL: flat_singlethread_release_acquire_cmpxchg:
3093; GFX11-CU:       ; %bb.0: ; %entry
3094; GFX11-CU-NEXT:    s_clause 0x1
3095; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3096; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3097; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3098; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3099; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3100; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3101; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3102; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3103; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3104; GFX11-CU-NEXT:    s_endpgm
3105    i32* %out, i32 %in, i32 %old) {
3106entry:
3107  %gep = getelementptr i32, i32* %out, i32 4
3108  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
3109  ret void
3110}
3111
3112define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_cmpxchg(
3113; GFX7-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
3114; GFX7:       ; %bb.0: ; %entry
3115; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3116; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3117; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3118; GFX7-NEXT:    s_add_u32 s0, s0, 16
3119; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3120; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3121; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3122; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3123; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3124; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3125; GFX7-NEXT:    s_endpgm
3126;
3127; GFX10-WGP-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
3128; GFX10-WGP:       ; %bb.0: ; %entry
3129; GFX10-WGP-NEXT:    s_clause 0x1
3130; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3131; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3132; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3133; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3134; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3135; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3136; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3137; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3138; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3139; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3140; GFX10-WGP-NEXT:    s_endpgm
3141;
3142; GFX10-CU-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
3143; GFX10-CU:       ; %bb.0: ; %entry
3144; GFX10-CU-NEXT:    s_clause 0x1
3145; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3146; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3147; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3148; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3149; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3150; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3151; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3152; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3153; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3154; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3155; GFX10-CU-NEXT:    s_endpgm
3156;
3157; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
3158; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3159; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3160; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3161; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3162; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3163; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3164; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3165; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3166; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3167; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3168; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3169; SKIP-CACHE-INV-NEXT:    s_endpgm
3170;
3171; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
3172; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3173; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3174; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3175; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3176; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3177; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3178; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3179; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3180;
3181; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
3182; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3183; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3184; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3185; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3186; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3187; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3188; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3189; GFX90A-TGSPLIT-NEXT:    s_endpgm
3190;
3191; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
3192; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3193; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3194; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3195; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3196; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3197; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3198; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3199; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3200;
3201; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
3202; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3203; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3204; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3205; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3206; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3207; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3208; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3209; GFX940-TGSPLIT-NEXT:    s_endpgm
3210;
3211; GFX11-WGP-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
3212; GFX11-WGP:       ; %bb.0: ; %entry
3213; GFX11-WGP-NEXT:    s_clause 0x1
3214; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3215; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3216; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3217; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3218; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3219; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3220; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3221; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3222; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3223; GFX11-WGP-NEXT:    s_endpgm
3224;
3225; GFX11-CU-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
3226; GFX11-CU:       ; %bb.0: ; %entry
3227; GFX11-CU-NEXT:    s_clause 0x1
3228; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3229; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3230; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3231; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3232; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3233; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3234; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3235; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3236; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3237; GFX11-CU-NEXT:    s_endpgm
3238    i32* %out, i32 %in, i32 %old) {
3239entry:
3240  %gep = getelementptr i32, i32* %out, i32 4
3241  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
3242  ret void
3243}
3244
3245define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_cmpxchg(
3246; GFX7-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
3247; GFX7:       ; %bb.0: ; %entry
3248; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3249; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3250; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3251; GFX7-NEXT:    s_add_u32 s0, s0, 16
3252; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3253; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3254; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3255; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3256; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3257; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3258; GFX7-NEXT:    s_endpgm
3259;
3260; GFX10-WGP-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
3261; GFX10-WGP:       ; %bb.0: ; %entry
3262; GFX10-WGP-NEXT:    s_clause 0x1
3263; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3264; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3265; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3266; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3267; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3268; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3269; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3270; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3271; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3272; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3273; GFX10-WGP-NEXT:    s_endpgm
3274;
3275; GFX10-CU-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
3276; GFX10-CU:       ; %bb.0: ; %entry
3277; GFX10-CU-NEXT:    s_clause 0x1
3278; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3279; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3280; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3281; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3282; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3283; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3284; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3285; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3286; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3287; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3288; GFX10-CU-NEXT:    s_endpgm
3289;
3290; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
3291; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3292; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3293; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3294; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3295; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3296; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3297; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3298; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3299; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3300; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3301; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3302; SKIP-CACHE-INV-NEXT:    s_endpgm
3303;
3304; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
3305; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3306; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3307; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3308; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3309; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3310; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3311; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3312; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3313;
3314; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
3315; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3316; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3317; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3318; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3319; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3320; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3321; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3322; GFX90A-TGSPLIT-NEXT:    s_endpgm
3323;
3324; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
3325; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3326; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3327; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3328; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3329; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3330; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3331; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3332; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3333;
3334; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
3335; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3336; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3337; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3338; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3339; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3340; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3341; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3342; GFX940-TGSPLIT-NEXT:    s_endpgm
3343;
3344; GFX11-WGP-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
3345; GFX11-WGP:       ; %bb.0: ; %entry
3346; GFX11-WGP-NEXT:    s_clause 0x1
3347; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3348; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3349; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3350; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3351; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3352; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3353; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3354; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3355; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3356; GFX11-WGP-NEXT:    s_endpgm
3357;
3358; GFX11-CU-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
3359; GFX11-CU:       ; %bb.0: ; %entry
3360; GFX11-CU-NEXT:    s_clause 0x1
3361; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3362; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3363; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3364; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3365; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3366; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3367; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3368; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3369; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3370; GFX11-CU-NEXT:    s_endpgm
3371    i32* %out, i32 %in, i32 %old) {
3372entry:
3373  %gep = getelementptr i32, i32* %out, i32 4
3374  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
3375  ret void
3376}
3377
3378define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_cmpxchg(
3379; GFX7-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
3380; GFX7:       ; %bb.0: ; %entry
3381; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3382; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3383; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3384; GFX7-NEXT:    s_add_u32 s0, s0, 16
3385; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3386; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3387; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3388; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3389; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3390; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3391; GFX7-NEXT:    s_endpgm
3392;
3393; GFX10-WGP-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
3394; GFX10-WGP:       ; %bb.0: ; %entry
3395; GFX10-WGP-NEXT:    s_clause 0x1
3396; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3397; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3398; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3399; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3400; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3401; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3402; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3403; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3404; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3405; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3406; GFX10-WGP-NEXT:    s_endpgm
3407;
3408; GFX10-CU-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
3409; GFX10-CU:       ; %bb.0: ; %entry
3410; GFX10-CU-NEXT:    s_clause 0x1
3411; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3412; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3413; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3414; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3415; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3416; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3417; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3418; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3419; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3420; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3421; GFX10-CU-NEXT:    s_endpgm
3422;
3423; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
3424; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3425; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3426; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3427; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3428; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3429; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3430; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3431; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3432; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3433; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3434; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3435; SKIP-CACHE-INV-NEXT:    s_endpgm
3436;
3437; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
3438; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3439; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3440; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3441; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3442; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3443; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3444; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3445; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3446;
3447; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
3448; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3449; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3450; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3451; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3452; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3453; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3454; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3455; GFX90A-TGSPLIT-NEXT:    s_endpgm
3456;
3457; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
3458; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3459; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3460; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3461; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3462; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3463; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3464; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3465; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3466;
3467; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
3468; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3469; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3470; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3471; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3472; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3473; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3474; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3475; GFX940-TGSPLIT-NEXT:    s_endpgm
3476;
3477; GFX11-WGP-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
3478; GFX11-WGP:       ; %bb.0: ; %entry
3479; GFX11-WGP-NEXT:    s_clause 0x1
3480; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3481; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3482; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3483; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3484; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3485; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3486; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3487; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3488; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3489; GFX11-WGP-NEXT:    s_endpgm
3490;
3491; GFX11-CU-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
3492; GFX11-CU:       ; %bb.0: ; %entry
3493; GFX11-CU-NEXT:    s_clause 0x1
3494; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3495; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3496; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3497; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3498; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3499; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3500; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3501; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3502; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3503; GFX11-CU-NEXT:    s_endpgm
3504    i32* %out, i32 %in, i32 %old) {
3505entry:
3506  %gep = getelementptr i32, i32* %out, i32 4
3507  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic seq_cst
3508  ret void
3509}
3510
3511define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_cmpxchg(
3512; GFX7-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
3513; GFX7:       ; %bb.0: ; %entry
3514; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3515; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3516; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3517; GFX7-NEXT:    s_add_u32 s0, s0, 16
3518; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3519; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3520; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3521; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3522; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3523; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3524; GFX7-NEXT:    s_endpgm
3525;
3526; GFX10-WGP-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
3527; GFX10-WGP:       ; %bb.0: ; %entry
3528; GFX10-WGP-NEXT:    s_clause 0x1
3529; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3530; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3531; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3532; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3533; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3534; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3535; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3536; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3537; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3538; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3539; GFX10-WGP-NEXT:    s_endpgm
3540;
3541; GFX10-CU-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
3542; GFX10-CU:       ; %bb.0: ; %entry
3543; GFX10-CU-NEXT:    s_clause 0x1
3544; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3545; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3546; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3547; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3548; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3549; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3550; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3551; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3552; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3553; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3554; GFX10-CU-NEXT:    s_endpgm
3555;
3556; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
3557; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3558; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3559; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3560; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3561; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3562; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3563; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3564; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3565; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3566; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3567; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3568; SKIP-CACHE-INV-NEXT:    s_endpgm
3569;
3570; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
3571; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3572; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3573; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3574; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3575; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3576; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3577; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3578; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3579;
3580; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
3581; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3582; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3583; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3584; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3585; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3586; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3587; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3588; GFX90A-TGSPLIT-NEXT:    s_endpgm
3589;
3590; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
3591; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3592; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3593; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3594; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3595; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3596; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3597; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3598; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3599;
3600; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
3601; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3602; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3603; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3604; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3605; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3606; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3607; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3608; GFX940-TGSPLIT-NEXT:    s_endpgm
3609;
3610; GFX11-WGP-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
3611; GFX11-WGP:       ; %bb.0: ; %entry
3612; GFX11-WGP-NEXT:    s_clause 0x1
3613; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3614; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3615; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3616; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3617; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3618; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3619; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3620; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3621; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3622; GFX11-WGP-NEXT:    s_endpgm
3623;
3624; GFX11-CU-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
3625; GFX11-CU:       ; %bb.0: ; %entry
3626; GFX11-CU-NEXT:    s_clause 0x1
3627; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3628; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3629; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3630; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3631; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3632; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3633; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3634; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3635; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3636; GFX11-CU-NEXT:    s_endpgm
3637    i32* %out, i32 %in, i32 %old) {
3638entry:
3639  %gep = getelementptr i32, i32* %out, i32 4
3640  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire seq_cst
3641  ret void
3642}
3643
3644define amdgpu_kernel void @flat_singlethread_release_seq_cst_cmpxchg(
3645; GFX7-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
3646; GFX7:       ; %bb.0: ; %entry
3647; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3648; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3649; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3650; GFX7-NEXT:    s_add_u32 s0, s0, 16
3651; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3652; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3653; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3654; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3655; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3656; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3657; GFX7-NEXT:    s_endpgm
3658;
3659; GFX10-WGP-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
3660; GFX10-WGP:       ; %bb.0: ; %entry
3661; GFX10-WGP-NEXT:    s_clause 0x1
3662; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3663; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3664; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3665; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3666; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3667; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3668; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3669; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3670; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3671; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3672; GFX10-WGP-NEXT:    s_endpgm
3673;
3674; GFX10-CU-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
3675; GFX10-CU:       ; %bb.0: ; %entry
3676; GFX10-CU-NEXT:    s_clause 0x1
3677; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3678; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3679; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3680; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3681; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3682; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3683; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3684; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3685; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3686; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3687; GFX10-CU-NEXT:    s_endpgm
3688;
3689; SKIP-CACHE-INV-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
3690; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3691; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3692; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3693; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3694; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3695; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3696; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3697; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3698; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3699; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3700; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3701; SKIP-CACHE-INV-NEXT:    s_endpgm
3702;
3703; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
3704; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3705; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3706; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3707; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3708; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3709; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3710; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3711; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3712;
3713; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
3714; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3715; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3716; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3717; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3718; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3719; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3720; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3721; GFX90A-TGSPLIT-NEXT:    s_endpgm
3722;
3723; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
3724; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3725; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3726; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3727; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3728; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3729; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3730; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3731; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3732;
3733; GFX940-TGSPLIT-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
3734; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3735; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3736; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3737; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3738; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3739; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3740; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3741; GFX940-TGSPLIT-NEXT:    s_endpgm
3742;
3743; GFX11-WGP-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
3744; GFX11-WGP:       ; %bb.0: ; %entry
3745; GFX11-WGP-NEXT:    s_clause 0x1
3746; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3747; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3748; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3749; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3750; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3751; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3752; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3753; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3754; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3755; GFX11-WGP-NEXT:    s_endpgm
3756;
3757; GFX11-CU-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
3758; GFX11-CU:       ; %bb.0: ; %entry
3759; GFX11-CU-NEXT:    s_clause 0x1
3760; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3761; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3762; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3763; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3764; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3765; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3766; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3767; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3768; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3769; GFX11-CU-NEXT:    s_endpgm
3770    i32* %out, i32 %in, i32 %old) {
3771entry:
3772  %gep = getelementptr i32, i32* %out, i32 4
3773  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release seq_cst
3774  ret void
3775}
3776
3777define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_cmpxchg(
3778; GFX7-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3779; GFX7:       ; %bb.0: ; %entry
3780; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3781; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3782; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3783; GFX7-NEXT:    s_add_u32 s0, s0, 16
3784; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3785; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3786; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3787; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3788; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3789; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3790; GFX7-NEXT:    s_endpgm
3791;
3792; GFX10-WGP-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3793; GFX10-WGP:       ; %bb.0: ; %entry
3794; GFX10-WGP-NEXT:    s_clause 0x1
3795; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3796; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3797; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3798; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3799; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3800; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3801; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3802; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3803; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3804; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3805; GFX10-WGP-NEXT:    s_endpgm
3806;
3807; GFX10-CU-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3808; GFX10-CU:       ; %bb.0: ; %entry
3809; GFX10-CU-NEXT:    s_clause 0x1
3810; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3811; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3812; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3813; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3814; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3815; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3816; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3817; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3818; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3819; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3820; GFX10-CU-NEXT:    s_endpgm
3821;
3822; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3823; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3824; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3825; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3826; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3827; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3828; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3829; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3830; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3831; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3832; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3833; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3834; SKIP-CACHE-INV-NEXT:    s_endpgm
3835;
3836; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3837; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3838; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3839; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3840; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3841; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3842; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3843; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3844; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3845;
3846; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3847; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3848; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3849; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3850; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3851; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3852; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3853; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3854; GFX90A-TGSPLIT-NEXT:    s_endpgm
3855;
3856; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3857; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3858; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3859; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3860; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3861; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3862; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3863; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3864; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3865;
3866; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3867; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3868; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3869; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3870; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3871; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3872; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3873; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3874; GFX940-TGSPLIT-NEXT:    s_endpgm
3875;
3876; GFX11-WGP-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3877; GFX11-WGP:       ; %bb.0: ; %entry
3878; GFX11-WGP-NEXT:    s_clause 0x1
3879; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3880; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3881; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3882; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3883; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3884; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3885; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3886; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3887; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3888; GFX11-WGP-NEXT:    s_endpgm
3889;
3890; GFX11-CU-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3891; GFX11-CU:       ; %bb.0: ; %entry
3892; GFX11-CU-NEXT:    s_clause 0x1
3893; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3894; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3895; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3896; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3897; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3898; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3899; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3900; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3901; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3902; GFX11-CU-NEXT:    s_endpgm
3903    i32* %out, i32 %in, i32 %old) {
3904entry:
3905  %gep = getelementptr i32, i32* %out, i32 4
3906  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel seq_cst
3907  ret void
3908}
3909
3910define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_cmpxchg(
3911; GFX7-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3912; GFX7:       ; %bb.0: ; %entry
3913; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3914; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3915; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3916; GFX7-NEXT:    s_add_u32 s0, s0, 16
3917; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3918; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3919; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3920; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3921; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3922; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3923; GFX7-NEXT:    s_endpgm
3924;
3925; GFX10-WGP-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3926; GFX10-WGP:       ; %bb.0: ; %entry
3927; GFX10-WGP-NEXT:    s_clause 0x1
3928; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3929; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3930; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3931; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3932; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3933; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3934; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3935; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3936; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3937; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3938; GFX10-WGP-NEXT:    s_endpgm
3939;
3940; GFX10-CU-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3941; GFX10-CU:       ; %bb.0: ; %entry
3942; GFX10-CU-NEXT:    s_clause 0x1
3943; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3944; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3945; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3946; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3947; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3948; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3949; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3950; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3951; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3952; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3953; GFX10-CU-NEXT:    s_endpgm
3954;
3955; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3956; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3957; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3958; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3959; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3960; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3961; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3962; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3963; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3964; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3965; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3966; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3967; SKIP-CACHE-INV-NEXT:    s_endpgm
3968;
3969; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3970; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3971; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3972; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3973; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3974; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3975; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3976; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3977; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3978;
3979; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3980; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3981; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3982; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3983; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3984; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3985; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3986; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3987; GFX90A-TGSPLIT-NEXT:    s_endpgm
3988;
3989; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3990; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3991; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3992; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3993; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3994; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3995; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3996; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3997; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3998;
3999; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
4000; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4001; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4002; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4003; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4004; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4005; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4006; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4007; GFX940-TGSPLIT-NEXT:    s_endpgm
4008;
4009; GFX11-WGP-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
4010; GFX11-WGP:       ; %bb.0: ; %entry
4011; GFX11-WGP-NEXT:    s_clause 0x1
4012; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4013; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4014; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4015; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4016; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4017; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4018; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4019; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4020; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4021; GFX11-WGP-NEXT:    s_endpgm
4022;
4023; GFX11-CU-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
4024; GFX11-CU:       ; %bb.0: ; %entry
4025; GFX11-CU-NEXT:    s_clause 0x1
4026; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4027; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4028; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4029; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4030; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4031; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4032; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4033; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4034; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4035; GFX11-CU-NEXT:    s_endpgm
4036    i32* %out, i32 %in, i32 %old) {
4037entry:
4038  %gep = getelementptr i32, i32* %out, i32 4
4039  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
4040  ret void
4041}
4042
4043define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_ret_cmpxchg(
4044; GFX7-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
4045; GFX7:       ; %bb.0: ; %entry
4046; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4047; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4048; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4049; GFX7-NEXT:    s_add_u32 s4, s0, 16
4050; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4051; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4052; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4053; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4054; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4055; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4056; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4057; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4058; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4059; GFX7-NEXT:    flat_store_dword v[0:1], v2
4060; GFX7-NEXT:    s_endpgm
4061;
4062; GFX10-WGP-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
4063; GFX10-WGP:       ; %bb.0: ; %entry
4064; GFX10-WGP-NEXT:    s_clause 0x1
4065; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4066; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4067; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4068; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4069; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4070; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4071; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4072; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4073; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4074; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4075; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4076; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4077; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4078; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4079; GFX10-WGP-NEXT:    s_endpgm
4080;
4081; GFX10-CU-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
4082; GFX10-CU:       ; %bb.0: ; %entry
4083; GFX10-CU-NEXT:    s_clause 0x1
4084; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4085; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4086; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4087; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4088; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4089; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4090; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4091; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4092; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4093; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4094; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4095; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4096; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4097; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4098; GFX10-CU-NEXT:    s_endpgm
4099;
4100; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
4101; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4102; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4103; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4104; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4105; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4106; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4107; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4108; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4109; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4110; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4111; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4112; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4113; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4114; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4115; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4116; SKIP-CACHE-INV-NEXT:    s_endpgm
4117;
4118; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
4119; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4120; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4121; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4122; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4123; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4124; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4125; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4126; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4127; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4128; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4129;
4130; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
4131; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4132; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4133; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4134; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4135; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4136; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4137; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4138; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4139; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4140; GFX90A-TGSPLIT-NEXT:    s_endpgm
4141;
4142; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
4143; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4144; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4145; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4146; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4147; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4148; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4149; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4150; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4151; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4152; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4153;
4154; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
4155; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4156; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4157; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4158; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4159; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4160; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4161; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4162; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4163; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4164; GFX940-TGSPLIT-NEXT:    s_endpgm
4165;
4166; GFX11-WGP-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
4167; GFX11-WGP:       ; %bb.0: ; %entry
4168; GFX11-WGP-NEXT:    s_clause 0x1
4169; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4170; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4171; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4172; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4173; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4174; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4175; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4176; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4177; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4178; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4179; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4180; GFX11-WGP-NEXT:    s_endpgm
4181;
4182; GFX11-CU-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
4183; GFX11-CU:       ; %bb.0: ; %entry
4184; GFX11-CU-NEXT:    s_clause 0x1
4185; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4186; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4187; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4188; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4189; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4190; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4191; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4192; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4193; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4194; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4195; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4196; GFX11-CU-NEXT:    s_endpgm
4197    i32* %out, i32 %in, i32 %old) {
4198entry:
4199  %gep = getelementptr i32, i32* %out, i32 4
4200  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
4201  %val0 = extractvalue { i32, i1 } %val, 0
4202  store i32 %val0, i32* %out, align 4
4203  ret void
4204}
4205
4206define amdgpu_kernel void @flat_singlethread_acquire_monotonic_ret_cmpxchg(
4207; GFX7-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
4208; GFX7:       ; %bb.0: ; %entry
4209; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4210; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4211; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4212; GFX7-NEXT:    s_add_u32 s4, s0, 16
4213; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4214; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4215; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4216; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4217; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4218; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4219; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4220; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4221; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4222; GFX7-NEXT:    flat_store_dword v[0:1], v2
4223; GFX7-NEXT:    s_endpgm
4224;
4225; GFX10-WGP-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
4226; GFX10-WGP:       ; %bb.0: ; %entry
4227; GFX10-WGP-NEXT:    s_clause 0x1
4228; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4229; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4230; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4231; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4232; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4233; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4234; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4235; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4236; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4237; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4238; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4239; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4240; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4241; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4242; GFX10-WGP-NEXT:    s_endpgm
4243;
4244; GFX10-CU-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
4245; GFX10-CU:       ; %bb.0: ; %entry
4246; GFX10-CU-NEXT:    s_clause 0x1
4247; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4248; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4249; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4250; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4251; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4252; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4253; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4254; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4255; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4256; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4257; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4258; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4259; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4260; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4261; GFX10-CU-NEXT:    s_endpgm
4262;
4263; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
4264; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4265; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4266; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4267; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4268; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4269; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4270; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4271; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4272; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4273; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4274; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4275; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4276; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4277; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4278; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4279; SKIP-CACHE-INV-NEXT:    s_endpgm
4280;
4281; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
4282; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4283; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4284; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4285; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4286; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4287; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4288; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4289; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4290; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4291; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4292;
4293; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
4294; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4295; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4296; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4297; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4298; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4299; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4300; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4301; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4302; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4303; GFX90A-TGSPLIT-NEXT:    s_endpgm
4304;
4305; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
4306; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4307; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4308; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4309; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4310; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4311; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4312; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4313; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4314; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4315; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4316;
4317; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
4318; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4319; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4320; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4321; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4322; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4323; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4324; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4325; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4326; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4327; GFX940-TGSPLIT-NEXT:    s_endpgm
4328;
4329; GFX11-WGP-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
4330; GFX11-WGP:       ; %bb.0: ; %entry
4331; GFX11-WGP-NEXT:    s_clause 0x1
4332; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4333; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4334; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4335; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4336; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4337; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4338; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4339; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4340; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4341; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4342; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4343; GFX11-WGP-NEXT:    s_endpgm
4344;
4345; GFX11-CU-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
4346; GFX11-CU:       ; %bb.0: ; %entry
4347; GFX11-CU-NEXT:    s_clause 0x1
4348; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4349; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4350; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4351; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4352; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4353; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4354; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4355; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4356; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4357; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4358; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4359; GFX11-CU-NEXT:    s_endpgm
4360    i32* %out, i32 %in, i32 %old) {
4361entry:
4362  %gep = getelementptr i32, i32* %out, i32 4
4363  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
4364  %val0 = extractvalue { i32, i1 } %val, 0
4365  store i32 %val0, i32* %out, align 4
4366  ret void
4367}
4368
4369define amdgpu_kernel void @flat_singlethread_release_monotonic_ret_cmpxchg(
4370; GFX7-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
4371; GFX7:       ; %bb.0: ; %entry
4372; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4373; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4374; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4375; GFX7-NEXT:    s_add_u32 s4, s0, 16
4376; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4377; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4378; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4379; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4380; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4381; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4382; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4383; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4384; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4385; GFX7-NEXT:    flat_store_dword v[0:1], v2
4386; GFX7-NEXT:    s_endpgm
4387;
4388; GFX10-WGP-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
4389; GFX10-WGP:       ; %bb.0: ; %entry
4390; GFX10-WGP-NEXT:    s_clause 0x1
4391; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4392; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4393; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4394; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4395; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4396; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4397; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4398; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4399; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4400; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4401; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4402; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4403; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4404; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4405; GFX10-WGP-NEXT:    s_endpgm
4406;
4407; GFX10-CU-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
4408; GFX10-CU:       ; %bb.0: ; %entry
4409; GFX10-CU-NEXT:    s_clause 0x1
4410; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4411; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4412; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4413; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4414; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4415; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4416; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4417; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4418; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4419; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4420; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4421; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4422; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4423; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4424; GFX10-CU-NEXT:    s_endpgm
4425;
4426; SKIP-CACHE-INV-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
4427; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4428; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4429; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4430; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4431; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4432; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4433; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4434; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4435; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4436; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4437; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4438; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4439; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4440; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4441; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4442; SKIP-CACHE-INV-NEXT:    s_endpgm
4443;
4444; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
4445; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4446; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4447; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4448; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4449; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4450; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4451; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4452; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4453; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4454; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4455;
4456; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
4457; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4458; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4459; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4460; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4461; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4462; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4463; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4464; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4465; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4466; GFX90A-TGSPLIT-NEXT:    s_endpgm
4467;
4468; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
4469; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4470; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4471; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4472; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4473; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4474; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4475; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4476; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4477; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4478; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4479;
4480; GFX940-TGSPLIT-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
4481; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4482; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4483; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4484; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4485; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4486; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4487; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4488; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4489; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4490; GFX940-TGSPLIT-NEXT:    s_endpgm
4491;
4492; GFX11-WGP-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
4493; GFX11-WGP:       ; %bb.0: ; %entry
4494; GFX11-WGP-NEXT:    s_clause 0x1
4495; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4496; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4497; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4498; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4499; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4500; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4501; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4502; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4503; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4504; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4505; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4506; GFX11-WGP-NEXT:    s_endpgm
4507;
4508; GFX11-CU-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
4509; GFX11-CU:       ; %bb.0: ; %entry
4510; GFX11-CU-NEXT:    s_clause 0x1
4511; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4512; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4513; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4514; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4515; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4516; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4517; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4518; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4519; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4520; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4521; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4522; GFX11-CU-NEXT:    s_endpgm
4523    i32* %out, i32 %in, i32 %old) {
4524entry:
4525  %gep = getelementptr i32, i32* %out, i32 4
4526  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
4527  %val0 = extractvalue { i32, i1 } %val, 0
4528  store i32 %val0, i32* %out, align 4
4529  ret void
4530}
4531
4532define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_ret_cmpxchg(
4533; GFX7-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
4534; GFX7:       ; %bb.0: ; %entry
4535; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4536; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4537; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4538; GFX7-NEXT:    s_add_u32 s4, s0, 16
4539; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4540; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4541; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4542; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4543; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4544; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4545; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4546; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4547; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4548; GFX7-NEXT:    flat_store_dword v[0:1], v2
4549; GFX7-NEXT:    s_endpgm
4550;
4551; GFX10-WGP-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
4552; GFX10-WGP:       ; %bb.0: ; %entry
4553; GFX10-WGP-NEXT:    s_clause 0x1
4554; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4555; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4556; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4557; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4558; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4559; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4560; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4561; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4562; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4563; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4564; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4565; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4566; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4567; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4568; GFX10-WGP-NEXT:    s_endpgm
4569;
4570; GFX10-CU-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
4571; GFX10-CU:       ; %bb.0: ; %entry
4572; GFX10-CU-NEXT:    s_clause 0x1
4573; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4574; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4575; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4576; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4577; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4578; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4579; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4580; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4581; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4582; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4583; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4584; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4585; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4586; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4587; GFX10-CU-NEXT:    s_endpgm
4588;
4589; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
4590; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4591; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4592; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4593; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4594; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4595; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4596; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4597; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4598; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4599; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4600; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4601; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4602; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4603; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4604; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4605; SKIP-CACHE-INV-NEXT:    s_endpgm
4606;
4607; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
4608; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4609; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4610; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4611; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4612; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4613; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4614; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4615; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4616; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4617; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4618;
4619; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
4620; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4621; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4622; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4623; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4624; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4625; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4626; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4627; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4628; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4629; GFX90A-TGSPLIT-NEXT:    s_endpgm
4630;
4631; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
4632; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4633; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4634; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4635; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4636; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4637; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4638; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4639; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4640; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4641; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4642;
4643; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
4644; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4645; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4646; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4647; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4648; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4649; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4650; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4651; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4652; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4653; GFX940-TGSPLIT-NEXT:    s_endpgm
4654;
4655; GFX11-WGP-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
4656; GFX11-WGP:       ; %bb.0: ; %entry
4657; GFX11-WGP-NEXT:    s_clause 0x1
4658; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4659; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4660; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4661; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4662; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4663; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4664; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4665; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4666; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4667; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4668; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4669; GFX11-WGP-NEXT:    s_endpgm
4670;
4671; GFX11-CU-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
4672; GFX11-CU:       ; %bb.0: ; %entry
4673; GFX11-CU-NEXT:    s_clause 0x1
4674; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4675; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4676; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4677; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4678; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4679; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4680; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4681; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4682; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4683; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4684; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4685; GFX11-CU-NEXT:    s_endpgm
4686    i32* %out, i32 %in, i32 %old) {
4687entry:
4688  %gep = getelementptr i32, i32* %out, i32 4
4689  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
4690  %val0 = extractvalue { i32, i1 } %val, 0
4691  store i32 %val0, i32* %out, align 4
4692  ret void
4693}
4694
4695define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_ret_cmpxchg(
4696; GFX7-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
4697; GFX7:       ; %bb.0: ; %entry
4698; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4699; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4700; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4701; GFX7-NEXT:    s_add_u32 s4, s0, 16
4702; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4703; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4704; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4705; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4706; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4707; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4708; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4709; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4710; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4711; GFX7-NEXT:    flat_store_dword v[0:1], v2
4712; GFX7-NEXT:    s_endpgm
4713;
4714; GFX10-WGP-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
4715; GFX10-WGP:       ; %bb.0: ; %entry
4716; GFX10-WGP-NEXT:    s_clause 0x1
4717; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4718; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4719; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4720; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4721; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4722; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4723; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4724; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4725; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4726; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4727; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4728; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4729; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4730; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4731; GFX10-WGP-NEXT:    s_endpgm
4732;
4733; GFX10-CU-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
4734; GFX10-CU:       ; %bb.0: ; %entry
4735; GFX10-CU-NEXT:    s_clause 0x1
4736; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4737; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4738; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4739; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4740; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4741; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4742; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4743; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4744; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4745; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4746; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4747; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4748; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4749; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4750; GFX10-CU-NEXT:    s_endpgm
4751;
4752; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
4753; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4754; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4755; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4756; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4757; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4758; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4759; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4760; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4761; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4762; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4763; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4764; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4765; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4766; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4767; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4768; SKIP-CACHE-INV-NEXT:    s_endpgm
4769;
4770; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
4771; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4772; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4773; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4774; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4775; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4776; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4777; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4778; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4779; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4780; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4781;
4782; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
4783; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4784; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4785; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4786; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4787; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4788; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4789; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4790; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4791; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4792; GFX90A-TGSPLIT-NEXT:    s_endpgm
4793;
4794; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
4795; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4796; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4797; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4798; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4799; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4800; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4801; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4802; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4803; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4804; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4805;
4806; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
4807; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4808; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4809; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4810; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4811; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4812; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4813; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4814; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4815; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4816; GFX940-TGSPLIT-NEXT:    s_endpgm
4817;
4818; GFX11-WGP-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
4819; GFX11-WGP:       ; %bb.0: ; %entry
4820; GFX11-WGP-NEXT:    s_clause 0x1
4821; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4822; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4823; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4824; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4825; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4826; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4827; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4828; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4829; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4830; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4831; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4832; GFX11-WGP-NEXT:    s_endpgm
4833;
4834; GFX11-CU-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
4835; GFX11-CU:       ; %bb.0: ; %entry
4836; GFX11-CU-NEXT:    s_clause 0x1
4837; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4838; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4839; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4840; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4841; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4842; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4843; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4844; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4845; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4846; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4847; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4848; GFX11-CU-NEXT:    s_endpgm
4849    i32* %out, i32 %in, i32 %old) {
4850entry:
4851  %gep = getelementptr i32, i32* %out, i32 4
4852  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
4853  %val0 = extractvalue { i32, i1 } %val, 0
4854  store i32 %val0, i32* %out, align 4
4855  ret void
4856}
4857
4858define amdgpu_kernel void @flat_singlethread_monotonic_acquire_ret_cmpxchg(
4859; GFX7-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
4860; GFX7:       ; %bb.0: ; %entry
4861; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4862; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4863; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4864; GFX7-NEXT:    s_add_u32 s4, s0, 16
4865; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4866; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4867; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4868; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4869; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4870; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4871; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4872; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4873; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4874; GFX7-NEXT:    flat_store_dword v[0:1], v2
4875; GFX7-NEXT:    s_endpgm
4876;
4877; GFX10-WGP-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
4878; GFX10-WGP:       ; %bb.0: ; %entry
4879; GFX10-WGP-NEXT:    s_clause 0x1
4880; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4881; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4882; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4883; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4884; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4885; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4886; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4887; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4888; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4889; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4890; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4891; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4892; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4893; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4894; GFX10-WGP-NEXT:    s_endpgm
4895;
4896; GFX10-CU-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
4897; GFX10-CU:       ; %bb.0: ; %entry
4898; GFX10-CU-NEXT:    s_clause 0x1
4899; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4900; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4901; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4902; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4903; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4904; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4905; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4906; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4907; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4908; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4909; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4910; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4911; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4912; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4913; GFX10-CU-NEXT:    s_endpgm
4914;
4915; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
4916; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4917; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4918; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4919; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4920; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4921; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4922; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4923; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4924; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4925; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4926; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4927; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4928; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4929; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4930; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4931; SKIP-CACHE-INV-NEXT:    s_endpgm
4932;
4933; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
4934; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4935; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4936; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4937; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4938; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4939; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4940; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4941; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4942; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4943; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4944;
4945; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
4946; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4947; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4948; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4949; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4950; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4951; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4952; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4953; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4954; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4955; GFX90A-TGSPLIT-NEXT:    s_endpgm
4956;
4957; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
4958; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4959; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4960; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4961; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4962; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4963; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4964; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4965; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4966; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4967; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4968;
4969; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
4970; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4971; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4972; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4973; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4974; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4975; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4976; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4977; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4978; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4979; GFX940-TGSPLIT-NEXT:    s_endpgm
4980;
4981; GFX11-WGP-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
4982; GFX11-WGP:       ; %bb.0: ; %entry
4983; GFX11-WGP-NEXT:    s_clause 0x1
4984; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4985; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4986; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4987; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4988; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4989; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4990; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4991; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4992; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4993; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4994; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4995; GFX11-WGP-NEXT:    s_endpgm
4996;
4997; GFX11-CU-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
4998; GFX11-CU:       ; %bb.0: ; %entry
4999; GFX11-CU-NEXT:    s_clause 0x1
5000; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5001; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5002; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5003; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5004; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5005; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5006; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5007; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5008; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5009; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5010; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5011; GFX11-CU-NEXT:    s_endpgm
5012    i32* %out, i32 %in, i32 %old) {
5013entry:
5014  %gep = getelementptr i32, i32* %out, i32 4
5015  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic acquire
5016  %val0 = extractvalue { i32, i1 } %val, 0
5017  store i32 %val0, i32* %out, align 4
5018  ret void
5019}
5020
5021define amdgpu_kernel void @flat_singlethread_acquire_acquire_ret_cmpxchg(
5022; GFX7-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
5023; GFX7:       ; %bb.0: ; %entry
5024; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5025; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5026; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5027; GFX7-NEXT:    s_add_u32 s4, s0, 16
5028; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5029; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5030; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5031; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5032; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5033; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5034; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5035; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5036; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5037; GFX7-NEXT:    flat_store_dword v[0:1], v2
5038; GFX7-NEXT:    s_endpgm
5039;
5040; GFX10-WGP-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
5041; GFX10-WGP:       ; %bb.0: ; %entry
5042; GFX10-WGP-NEXT:    s_clause 0x1
5043; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5044; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5045; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5046; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5047; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5048; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5049; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5050; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5051; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5052; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5053; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5054; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5055; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5056; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5057; GFX10-WGP-NEXT:    s_endpgm
5058;
5059; GFX10-CU-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
5060; GFX10-CU:       ; %bb.0: ; %entry
5061; GFX10-CU-NEXT:    s_clause 0x1
5062; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5063; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5064; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5065; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5066; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5067; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5068; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5069; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5070; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5071; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5072; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5073; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5074; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5075; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5076; GFX10-CU-NEXT:    s_endpgm
5077;
5078; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
5079; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5080; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5081; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5082; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5083; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5084; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5085; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5086; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5087; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5088; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5089; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5090; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5091; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5092; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5093; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5094; SKIP-CACHE-INV-NEXT:    s_endpgm
5095;
5096; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
5097; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5098; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5099; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5100; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5101; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5102; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5103; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5104; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5105; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5106; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5107;
5108; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
5109; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5110; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5111; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5112; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5113; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5114; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5115; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5116; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5117; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5118; GFX90A-TGSPLIT-NEXT:    s_endpgm
5119;
5120; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
5121; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5122; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5123; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5124; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5125; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5126; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5127; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5128; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5129; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5130; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5131;
5132; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
5133; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5134; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5135; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5136; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5137; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5138; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5139; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5140; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5141; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5142; GFX940-TGSPLIT-NEXT:    s_endpgm
5143;
5144; GFX11-WGP-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
5145; GFX11-WGP:       ; %bb.0: ; %entry
5146; GFX11-WGP-NEXT:    s_clause 0x1
5147; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5148; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5149; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5150; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5151; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5152; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5153; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5154; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5155; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5156; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5157; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5158; GFX11-WGP-NEXT:    s_endpgm
5159;
5160; GFX11-CU-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
5161; GFX11-CU:       ; %bb.0: ; %entry
5162; GFX11-CU-NEXT:    s_clause 0x1
5163; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5164; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5165; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5166; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5167; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5168; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5169; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5170; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5171; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5172; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5173; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5174; GFX11-CU-NEXT:    s_endpgm
5175    i32* %out, i32 %in, i32 %old) {
5176entry:
5177  %gep = getelementptr i32, i32* %out, i32 4
5178  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
5179  %val0 = extractvalue { i32, i1 } %val, 0
5180  store i32 %val0, i32* %out, align 4
5181  ret void
5182}
5183
5184define amdgpu_kernel void @flat_singlethread_release_acquire_ret_cmpxchg(
5185; GFX7-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
5186; GFX7:       ; %bb.0: ; %entry
5187; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5188; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5189; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5190; GFX7-NEXT:    s_add_u32 s4, s0, 16
5191; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5192; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5193; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5194; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5195; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5196; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5197; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5198; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5199; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5200; GFX7-NEXT:    flat_store_dword v[0:1], v2
5201; GFX7-NEXT:    s_endpgm
5202;
5203; GFX10-WGP-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
5204; GFX10-WGP:       ; %bb.0: ; %entry
5205; GFX10-WGP-NEXT:    s_clause 0x1
5206; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5207; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5208; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5209; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5210; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5211; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5212; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5213; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5214; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5215; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5216; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5217; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5218; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5219; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5220; GFX10-WGP-NEXT:    s_endpgm
5221;
5222; GFX10-CU-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
5223; GFX10-CU:       ; %bb.0: ; %entry
5224; GFX10-CU-NEXT:    s_clause 0x1
5225; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5226; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5227; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5228; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5229; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5230; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5231; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5232; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5233; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5234; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5235; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5236; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5237; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5238; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5239; GFX10-CU-NEXT:    s_endpgm
5240;
5241; SKIP-CACHE-INV-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
5242; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5243; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5244; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5245; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5246; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5247; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5248; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5249; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5250; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5251; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5252; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5253; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5254; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5255; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5256; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5257; SKIP-CACHE-INV-NEXT:    s_endpgm
5258;
5259; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
5260; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5261; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5262; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5263; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5264; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5265; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5266; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5267; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5268; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5269; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5270;
5271; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
5272; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5273; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5274; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5275; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5276; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5277; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5278; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5279; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5280; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5281; GFX90A-TGSPLIT-NEXT:    s_endpgm
5282;
5283; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
5284; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5285; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5286; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5287; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5288; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5289; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5290; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5291; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5292; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5293; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5294;
5295; GFX940-TGSPLIT-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
5296; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5297; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5298; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5299; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5300; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5301; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5302; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5303; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5304; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5305; GFX940-TGSPLIT-NEXT:    s_endpgm
5306;
5307; GFX11-WGP-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
5308; GFX11-WGP:       ; %bb.0: ; %entry
5309; GFX11-WGP-NEXT:    s_clause 0x1
5310; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5311; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5312; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5313; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5314; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5315; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5316; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5317; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5318; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5319; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5320; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5321; GFX11-WGP-NEXT:    s_endpgm
5322;
5323; GFX11-CU-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
5324; GFX11-CU:       ; %bb.0: ; %entry
5325; GFX11-CU-NEXT:    s_clause 0x1
5326; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5327; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5328; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5329; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5330; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5331; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5332; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5333; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5334; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5335; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5336; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5337; GFX11-CU-NEXT:    s_endpgm
5338    i32* %out, i32 %in, i32 %old) {
5339entry:
5340  %gep = getelementptr i32, i32* %out, i32 4
5341  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
5342  %val0 = extractvalue { i32, i1 } %val, 0
5343  store i32 %val0, i32* %out, align 4
5344  ret void
5345}
5346
5347define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_ret_cmpxchg(
5348; GFX7-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
5349; GFX7:       ; %bb.0: ; %entry
5350; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5351; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5352; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5353; GFX7-NEXT:    s_add_u32 s4, s0, 16
5354; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5355; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5356; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5357; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5358; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5359; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5360; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5361; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5362; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5363; GFX7-NEXT:    flat_store_dword v[0:1], v2
5364; GFX7-NEXT:    s_endpgm
5365;
5366; GFX10-WGP-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
5367; GFX10-WGP:       ; %bb.0: ; %entry
5368; GFX10-WGP-NEXT:    s_clause 0x1
5369; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5370; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5371; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5372; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5373; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5374; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5375; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5376; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5377; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5378; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5379; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5380; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5381; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5382; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5383; GFX10-WGP-NEXT:    s_endpgm
5384;
5385; GFX10-CU-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
5386; GFX10-CU:       ; %bb.0: ; %entry
5387; GFX10-CU-NEXT:    s_clause 0x1
5388; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5389; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5390; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5391; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5392; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5393; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5394; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5395; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5396; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5397; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5398; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5399; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5400; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5401; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5402; GFX10-CU-NEXT:    s_endpgm
5403;
5404; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
5405; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5406; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5407; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5408; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5409; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5410; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5411; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5412; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5413; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5414; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5415; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5416; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5417; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5418; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5419; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5420; SKIP-CACHE-INV-NEXT:    s_endpgm
5421;
5422; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
5423; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5424; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5425; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5426; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5427; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5428; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5429; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5430; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5431; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5432; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5433;
5434; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
5435; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5436; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5437; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5438; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5439; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5440; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5441; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5442; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5443; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5444; GFX90A-TGSPLIT-NEXT:    s_endpgm
5445;
5446; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
5447; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5448; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5449; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5450; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5451; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5452; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5453; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5454; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5455; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5456; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5457;
5458; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
5459; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5460; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5461; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5462; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5463; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5464; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5465; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5466; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5467; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5468; GFX940-TGSPLIT-NEXT:    s_endpgm
5469;
5470; GFX11-WGP-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
5471; GFX11-WGP:       ; %bb.0: ; %entry
5472; GFX11-WGP-NEXT:    s_clause 0x1
5473; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5474; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5475; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5476; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5477; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5478; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5479; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5480; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5481; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5482; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5483; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5484; GFX11-WGP-NEXT:    s_endpgm
5485;
5486; GFX11-CU-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
5487; GFX11-CU:       ; %bb.0: ; %entry
5488; GFX11-CU-NEXT:    s_clause 0x1
5489; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5490; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5491; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5492; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5493; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5494; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5495; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5496; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5497; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5498; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5499; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5500; GFX11-CU-NEXT:    s_endpgm
5501    i32* %out, i32 %in, i32 %old) {
5502entry:
5503  %gep = getelementptr i32, i32* %out, i32 4
5504  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
5505  %val0 = extractvalue { i32, i1 } %val, 0
5506  store i32 %val0, i32* %out, align 4
5507  ret void
5508}
5509
5510define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_ret_cmpxchg(
5511; GFX7-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
5512; GFX7:       ; %bb.0: ; %entry
5513; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5514; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5515; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5516; GFX7-NEXT:    s_add_u32 s4, s0, 16
5517; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5518; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5519; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5520; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5521; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5522; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5523; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5524; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5525; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5526; GFX7-NEXT:    flat_store_dword v[0:1], v2
5527; GFX7-NEXT:    s_endpgm
5528;
5529; GFX10-WGP-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
5530; GFX10-WGP:       ; %bb.0: ; %entry
5531; GFX10-WGP-NEXT:    s_clause 0x1
5532; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5533; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5534; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5535; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5536; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5537; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5538; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5539; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5540; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5541; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5542; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5543; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5544; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5545; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5546; GFX10-WGP-NEXT:    s_endpgm
5547;
5548; GFX10-CU-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
5549; GFX10-CU:       ; %bb.0: ; %entry
5550; GFX10-CU-NEXT:    s_clause 0x1
5551; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5552; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5553; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5554; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5555; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5556; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5557; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5558; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5559; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5560; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5561; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5562; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5563; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5564; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5565; GFX10-CU-NEXT:    s_endpgm
5566;
5567; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
5568; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5569; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5570; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5571; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5572; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5573; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5574; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5575; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5576; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5577; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5578; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5579; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5580; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5581; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5582; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5583; SKIP-CACHE-INV-NEXT:    s_endpgm
5584;
5585; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
5586; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5587; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5588; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5589; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5590; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5591; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5592; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5593; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5594; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5595; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5596;
5597; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
5598; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5599; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5600; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5601; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5602; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5603; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5604; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5605; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5606; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5607; GFX90A-TGSPLIT-NEXT:    s_endpgm
5608;
5609; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
5610; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5611; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5612; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5613; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5614; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5615; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5616; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5617; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5618; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5619; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5620;
5621; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
5622; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5623; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5624; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5625; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5626; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5627; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5628; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5629; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5630; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5631; GFX940-TGSPLIT-NEXT:    s_endpgm
5632;
5633; GFX11-WGP-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
5634; GFX11-WGP:       ; %bb.0: ; %entry
5635; GFX11-WGP-NEXT:    s_clause 0x1
5636; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5637; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5638; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5639; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5640; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5641; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5642; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5643; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5644; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5645; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5646; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5647; GFX11-WGP-NEXT:    s_endpgm
5648;
5649; GFX11-CU-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
5650; GFX11-CU:       ; %bb.0: ; %entry
5651; GFX11-CU-NEXT:    s_clause 0x1
5652; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5653; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5654; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5655; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5656; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5657; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5658; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5659; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5660; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5661; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5662; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5663; GFX11-CU-NEXT:    s_endpgm
5664    i32* %out, i32 %in, i32 %old) {
5665entry:
5666  %gep = getelementptr i32, i32* %out, i32 4
5667  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
5668  %val0 = extractvalue { i32, i1 } %val, 0
5669  store i32 %val0, i32* %out, align 4
5670  ret void
5671}
5672
5673define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_ret_cmpxchg(
5674; GFX7-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
5675; GFX7:       ; %bb.0: ; %entry
5676; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5677; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5678; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5679; GFX7-NEXT:    s_add_u32 s4, s0, 16
5680; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5681; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5682; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5683; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5684; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5685; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5686; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5687; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5688; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5689; GFX7-NEXT:    flat_store_dword v[0:1], v2
5690; GFX7-NEXT:    s_endpgm
5691;
5692; GFX10-WGP-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
5693; GFX10-WGP:       ; %bb.0: ; %entry
5694; GFX10-WGP-NEXT:    s_clause 0x1
5695; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5696; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5697; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5698; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5699; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5700; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5701; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5702; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5703; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5704; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5705; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5706; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5707; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5708; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5709; GFX10-WGP-NEXT:    s_endpgm
5710;
5711; GFX10-CU-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
5712; GFX10-CU:       ; %bb.0: ; %entry
5713; GFX10-CU-NEXT:    s_clause 0x1
5714; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5715; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5716; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5717; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5718; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5719; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5720; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5721; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5722; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5723; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5724; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5725; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5726; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5727; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5728; GFX10-CU-NEXT:    s_endpgm
5729;
5730; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
5731; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5732; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5733; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5734; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5735; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5736; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5737; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5738; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5739; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5740; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5741; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5742; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5743; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5744; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5745; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5746; SKIP-CACHE-INV-NEXT:    s_endpgm
5747;
5748; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
5749; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5750; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5751; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5752; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5753; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5754; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5755; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5756; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5757; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5758; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5759;
5760; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
5761; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5762; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5763; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5764; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5765; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5766; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5767; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5768; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5769; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5770; GFX90A-TGSPLIT-NEXT:    s_endpgm
5771;
5772; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
5773; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5774; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5775; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5776; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5777; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5778; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5779; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5780; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5781; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5782; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5783;
5784; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
5785; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5786; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5787; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5788; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5789; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5790; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5791; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5792; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5793; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5794; GFX940-TGSPLIT-NEXT:    s_endpgm
5795;
5796; GFX11-WGP-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
5797; GFX11-WGP:       ; %bb.0: ; %entry
5798; GFX11-WGP-NEXT:    s_clause 0x1
5799; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5800; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5801; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5802; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5803; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5804; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5805; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5806; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5807; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5808; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5809; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5810; GFX11-WGP-NEXT:    s_endpgm
5811;
5812; GFX11-CU-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
5813; GFX11-CU:       ; %bb.0: ; %entry
5814; GFX11-CU-NEXT:    s_clause 0x1
5815; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5816; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5817; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5818; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5819; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5820; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5821; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5822; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5823; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5824; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5825; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5826; GFX11-CU-NEXT:    s_endpgm
5827    i32* %out, i32 %in, i32 %old) {
5828entry:
5829  %gep = getelementptr i32, i32* %out, i32 4
5830  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic seq_cst
5831  %val0 = extractvalue { i32, i1 } %val, 0
5832  store i32 %val0, i32* %out, align 4
5833  ret void
5834}
5835
5836define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_ret_cmpxchg(
5837; GFX7-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
5838; GFX7:       ; %bb.0: ; %entry
5839; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5840; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5841; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5842; GFX7-NEXT:    s_add_u32 s4, s0, 16
5843; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5844; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5845; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5846; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5847; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5848; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5849; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5850; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5851; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5852; GFX7-NEXT:    flat_store_dword v[0:1], v2
5853; GFX7-NEXT:    s_endpgm
5854;
5855; GFX10-WGP-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
5856; GFX10-WGP:       ; %bb.0: ; %entry
5857; GFX10-WGP-NEXT:    s_clause 0x1
5858; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5859; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5860; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5861; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5862; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5863; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5864; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5865; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5866; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5867; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5868; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5869; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5870; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5871; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5872; GFX10-WGP-NEXT:    s_endpgm
5873;
5874; GFX10-CU-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
5875; GFX10-CU:       ; %bb.0: ; %entry
5876; GFX10-CU-NEXT:    s_clause 0x1
5877; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5878; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5879; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5880; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5881; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5882; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5883; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5884; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5885; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5886; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5887; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5888; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5889; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5890; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5891; GFX10-CU-NEXT:    s_endpgm
5892;
5893; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
5894; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5895; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5896; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5897; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5898; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5899; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5900; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5901; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5902; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5903; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5904; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5905; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5906; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5907; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5908; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5909; SKIP-CACHE-INV-NEXT:    s_endpgm
5910;
5911; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
5912; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5913; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5914; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5915; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5916; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5917; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5918; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5919; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5920; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5921; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5922;
5923; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
5924; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5925; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5926; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5927; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5928; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5929; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5930; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5931; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5932; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5933; GFX90A-TGSPLIT-NEXT:    s_endpgm
5934;
5935; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
5936; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5937; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5938; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5939; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5940; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5941; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5942; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5943; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5944; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5945; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5946;
5947; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
5948; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5949; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5950; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5951; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5952; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5953; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5954; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5955; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5956; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5957; GFX940-TGSPLIT-NEXT:    s_endpgm
5958;
5959; GFX11-WGP-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
5960; GFX11-WGP:       ; %bb.0: ; %entry
5961; GFX11-WGP-NEXT:    s_clause 0x1
5962; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5963; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5964; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5965; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5966; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5967; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5968; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5969; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5970; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5971; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5972; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5973; GFX11-WGP-NEXT:    s_endpgm
5974;
5975; GFX11-CU-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
5976; GFX11-CU:       ; %bb.0: ; %entry
5977; GFX11-CU-NEXT:    s_clause 0x1
5978; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5979; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5980; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5981; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5982; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5983; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5984; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5985; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5986; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5987; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5988; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5989; GFX11-CU-NEXT:    s_endpgm
5990    i32* %out, i32 %in, i32 %old) {
5991entry:
5992  %gep = getelementptr i32, i32* %out, i32 4
5993  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire seq_cst
5994  %val0 = extractvalue { i32, i1 } %val, 0
5995  store i32 %val0, i32* %out, align 4
5996  ret void
5997}
5998
5999define amdgpu_kernel void @flat_singlethread_release_seq_cst_ret_cmpxchg(
6000; GFX7-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
6001; GFX7:       ; %bb.0: ; %entry
6002; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6003; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6004; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6005; GFX7-NEXT:    s_add_u32 s4, s0, 16
6006; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6007; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6008; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6009; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6010; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6011; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6012; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6013; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6014; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6015; GFX7-NEXT:    flat_store_dword v[0:1], v2
6016; GFX7-NEXT:    s_endpgm
6017;
6018; GFX10-WGP-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
6019; GFX10-WGP:       ; %bb.0: ; %entry
6020; GFX10-WGP-NEXT:    s_clause 0x1
6021; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6022; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6023; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6024; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6025; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6026; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6027; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6028; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6029; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6030; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6031; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6032; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6033; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6034; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6035; GFX10-WGP-NEXT:    s_endpgm
6036;
6037; GFX10-CU-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
6038; GFX10-CU:       ; %bb.0: ; %entry
6039; GFX10-CU-NEXT:    s_clause 0x1
6040; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6041; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6042; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6043; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6044; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6045; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6046; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6047; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6048; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6049; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6050; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6051; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6052; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6053; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6054; GFX10-CU-NEXT:    s_endpgm
6055;
6056; SKIP-CACHE-INV-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
6057; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6058; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6059; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6060; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6061; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6062; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6063; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6064; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6065; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6066; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6067; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6068; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6069; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6070; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6071; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6072; SKIP-CACHE-INV-NEXT:    s_endpgm
6073;
6074; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
6075; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6076; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6077; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6078; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6079; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6080; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6081; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6082; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6083; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6084; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6085;
6086; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
6087; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6088; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6089; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6090; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6091; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6092; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6093; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6094; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6095; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6096; GFX90A-TGSPLIT-NEXT:    s_endpgm
6097;
6098; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
6099; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6100; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6101; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6102; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6103; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6104; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6105; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6106; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6107; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6108; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6109;
6110; GFX940-TGSPLIT-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
6111; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6112; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6113; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6114; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6115; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6116; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6117; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6118; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6119; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6120; GFX940-TGSPLIT-NEXT:    s_endpgm
6121;
6122; GFX11-WGP-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
6123; GFX11-WGP:       ; %bb.0: ; %entry
6124; GFX11-WGP-NEXT:    s_clause 0x1
6125; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6126; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6127; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6128; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6129; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
6130; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6131; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
6132; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6133; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6134; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6135; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6136; GFX11-WGP-NEXT:    s_endpgm
6137;
6138; GFX11-CU-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
6139; GFX11-CU:       ; %bb.0: ; %entry
6140; GFX11-CU-NEXT:    s_clause 0x1
6141; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6142; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6143; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6144; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6145; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
6146; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6147; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6148; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6149; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6150; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6151; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6152; GFX11-CU-NEXT:    s_endpgm
6153    i32* %out, i32 %in, i32 %old) {
6154entry:
6155  %gep = getelementptr i32, i32* %out, i32 4
6156  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release seq_cst
6157  %val0 = extractvalue { i32, i1 } %val, 0
6158  store i32 %val0, i32* %out, align 4
6159  ret void
6160}
6161
6162define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_ret_cmpxchg(
6163; GFX7-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6164; GFX7:       ; %bb.0: ; %entry
6165; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6166; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6167; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6168; GFX7-NEXT:    s_add_u32 s4, s0, 16
6169; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6170; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6171; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6172; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6173; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6174; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6175; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6176; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6177; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6178; GFX7-NEXT:    flat_store_dword v[0:1], v2
6179; GFX7-NEXT:    s_endpgm
6180;
6181; GFX10-WGP-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6182; GFX10-WGP:       ; %bb.0: ; %entry
6183; GFX10-WGP-NEXT:    s_clause 0x1
6184; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6185; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6186; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6187; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6188; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6189; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6190; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6191; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6192; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6193; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6194; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6195; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6196; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6197; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6198; GFX10-WGP-NEXT:    s_endpgm
6199;
6200; GFX10-CU-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6201; GFX10-CU:       ; %bb.0: ; %entry
6202; GFX10-CU-NEXT:    s_clause 0x1
6203; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6204; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6205; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6206; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6207; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6208; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6209; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6210; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6211; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6212; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6213; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6214; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6215; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6216; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6217; GFX10-CU-NEXT:    s_endpgm
6218;
6219; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6220; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6221; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6222; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6223; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6224; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6225; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6226; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6227; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6228; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6229; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6230; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6231; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6232; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6233; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6234; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6235; SKIP-CACHE-INV-NEXT:    s_endpgm
6236;
6237; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6238; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6239; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6240; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6241; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6242; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6243; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6244; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6245; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6246; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6247; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6248;
6249; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6250; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6251; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6252; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6253; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6254; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6255; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6256; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6257; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6258; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6259; GFX90A-TGSPLIT-NEXT:    s_endpgm
6260;
6261; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6262; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6263; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6264; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6265; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6266; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6267; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6268; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6269; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6270; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6271; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6272;
6273; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6274; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6275; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6276; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6277; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6278; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6279; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6280; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6281; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6282; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6283; GFX940-TGSPLIT-NEXT:    s_endpgm
6284;
6285; GFX11-WGP-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6286; GFX11-WGP:       ; %bb.0: ; %entry
6287; GFX11-WGP-NEXT:    s_clause 0x1
6288; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6289; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6290; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6291; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6292; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
6293; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6294; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
6295; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6296; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6297; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6298; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6299; GFX11-WGP-NEXT:    s_endpgm
6300;
6301; GFX11-CU-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6302; GFX11-CU:       ; %bb.0: ; %entry
6303; GFX11-CU-NEXT:    s_clause 0x1
6304; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6305; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6306; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6307; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6308; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
6309; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6310; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6311; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6312; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6313; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6314; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6315; GFX11-CU-NEXT:    s_endpgm
6316    i32* %out, i32 %in, i32 %old) {
6317entry:
6318  %gep = getelementptr i32, i32* %out, i32 4
6319  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel seq_cst
6320  %val0 = extractvalue { i32, i1 } %val, 0
6321  store i32 %val0, i32* %out, align 4
6322  ret void
6323}
6324
6325define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_ret_cmpxchg(
6326; GFX7-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6327; GFX7:       ; %bb.0: ; %entry
6328; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6329; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6330; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6331; GFX7-NEXT:    s_add_u32 s4, s0, 16
6332; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6333; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6334; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6335; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6336; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6337; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6338; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6339; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6340; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6341; GFX7-NEXT:    flat_store_dword v[0:1], v2
6342; GFX7-NEXT:    s_endpgm
6343;
6344; GFX10-WGP-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6345; GFX10-WGP:       ; %bb.0: ; %entry
6346; GFX10-WGP-NEXT:    s_clause 0x1
6347; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6348; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6349; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6350; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6351; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6352; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6353; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6354; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6355; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6356; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6357; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6358; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6359; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6360; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6361; GFX10-WGP-NEXT:    s_endpgm
6362;
6363; GFX10-CU-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6364; GFX10-CU:       ; %bb.0: ; %entry
6365; GFX10-CU-NEXT:    s_clause 0x1
6366; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6367; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6368; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6369; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6370; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6371; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6372; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6373; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6374; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6375; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6376; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6377; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6378; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6379; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6380; GFX10-CU-NEXT:    s_endpgm
6381;
6382; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6383; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6384; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6385; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6386; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6387; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6388; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6389; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6390; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6391; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6392; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6393; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6394; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6395; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6396; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6397; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6398; SKIP-CACHE-INV-NEXT:    s_endpgm
6399;
6400; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6401; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6402; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6403; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6404; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6405; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6406; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6407; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6408; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6409; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6410; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6411;
6412; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6413; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6414; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6415; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6416; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6417; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6418; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6419; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6420; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6421; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6422; GFX90A-TGSPLIT-NEXT:    s_endpgm
6423;
6424; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6425; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6426; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6427; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6428; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6429; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6430; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6431; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6432; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6433; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6434; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6435;
6436; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6437; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6438; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6439; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6440; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6441; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6442; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6443; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6444; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6445; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6446; GFX940-TGSPLIT-NEXT:    s_endpgm
6447;
6448; GFX11-WGP-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6449; GFX11-WGP:       ; %bb.0: ; %entry
6450; GFX11-WGP-NEXT:    s_clause 0x1
6451; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6452; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6453; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6454; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6455; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
6456; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6457; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
6458; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6459; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6460; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6461; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6462; GFX11-WGP-NEXT:    s_endpgm
6463;
6464; GFX11-CU-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6465; GFX11-CU:       ; %bb.0: ; %entry
6466; GFX11-CU-NEXT:    s_clause 0x1
6467; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6468; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6469; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6470; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6471; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
6472; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6473; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6474; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6475; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6476; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6477; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6478; GFX11-CU-NEXT:    s_endpgm
6479    i32* %out, i32 %in, i32 %old) {
6480entry:
6481  %gep = getelementptr i32, i32* %out, i32 4
6482  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
6483  %val0 = extractvalue { i32, i1 } %val, 0
6484  store i32 %val0, i32* %out, align 4
6485  ret void
6486}
6487
6488define amdgpu_kernel void @flat_singlethread_one_as_unordered_load(
6489; GFX7-LABEL: flat_singlethread_one_as_unordered_load:
6490; GFX7:       ; %bb.0: ; %entry
6491; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6492; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6493; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6494; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6495; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6496; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6497; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6498; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6499; GFX7-NEXT:    flat_store_dword v[0:1], v2
6500; GFX7-NEXT:    s_endpgm
6501;
6502; GFX10-WGP-LABEL: flat_singlethread_one_as_unordered_load:
6503; GFX10-WGP:       ; %bb.0: ; %entry
6504; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6505; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6506; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6507; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6508; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
6509; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6510; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6511; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6512; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6513; GFX10-WGP-NEXT:    s_endpgm
6514;
6515; GFX10-CU-LABEL: flat_singlethread_one_as_unordered_load:
6516; GFX10-CU:       ; %bb.0: ; %entry
6517; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6518; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6519; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6520; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6521; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6522; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6523; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6524; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6525; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6526; GFX10-CU-NEXT:    s_endpgm
6527;
6528; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_unordered_load:
6529; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6530; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6531; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6532; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6533; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6534; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
6535; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6536; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6537; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6538; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6539; SKIP-CACHE-INV-NEXT:    s_endpgm
6540;
6541; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_unordered_load:
6542; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6543; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6544; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6545; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6546; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6547; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6548; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6549; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6550; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6551; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6552; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6553;
6554; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_unordered_load:
6555; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6556; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6557; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6558; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6559; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6560; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6561; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6562; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6563; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6564; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6565; GFX90A-TGSPLIT-NEXT:    s_endpgm
6566;
6567; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_unordered_load:
6568; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6569; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6570; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6571; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6572; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6573; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6574; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6575; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6576; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6577; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6578; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6579;
6580; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_unordered_load:
6581; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6582; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6583; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6584; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6585; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6586; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6587; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6588; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6589; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6590; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6591; GFX940-TGSPLIT-NEXT:    s_endpgm
6592;
6593; GFX11-WGP-LABEL: flat_singlethread_one_as_unordered_load:
6594; GFX11-WGP:       ; %bb.0: ; %entry
6595; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6596; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6597; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
6598; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
6599; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
6600; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6601; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6602; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6603; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6604; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6605; GFX11-WGP-NEXT:    s_endpgm
6606;
6607; GFX11-CU-LABEL: flat_singlethread_one_as_unordered_load:
6608; GFX11-CU:       ; %bb.0: ; %entry
6609; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6610; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6611; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
6612; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
6613; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
6614; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6615; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6616; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6617; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6618; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6619; GFX11-CU-NEXT:    s_endpgm
6620    i32* %in, i32* %out) {
6621entry:
6622  %val = load atomic i32, i32* %in syncscope("singlethread-one-as") unordered, align 4
6623  store i32 %val, i32* %out
6624  ret void
6625}
6626
6627define amdgpu_kernel void @flat_singlethread_one_as_monotonic_load(
6628; GFX7-LABEL: flat_singlethread_one_as_monotonic_load:
6629; GFX7:       ; %bb.0: ; %entry
6630; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6631; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6632; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6633; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6634; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6635; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6636; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6637; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6638; GFX7-NEXT:    flat_store_dword v[0:1], v2
6639; GFX7-NEXT:    s_endpgm
6640;
6641; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_load:
6642; GFX10-WGP:       ; %bb.0: ; %entry
6643; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6644; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6645; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6646; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6647; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
6648; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6649; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6650; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6651; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6652; GFX10-WGP-NEXT:    s_endpgm
6653;
6654; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_load:
6655; GFX10-CU:       ; %bb.0: ; %entry
6656; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6657; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6658; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6659; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6660; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6661; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6662; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6663; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6664; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6665; GFX10-CU-NEXT:    s_endpgm
6666;
6667; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_load:
6668; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6669; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6670; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6671; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6672; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6673; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
6674; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6675; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6676; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6677; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6678; SKIP-CACHE-INV-NEXT:    s_endpgm
6679;
6680; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_load:
6681; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6682; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6683; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6684; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6685; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6686; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6687; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6688; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6689; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6690; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6691; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6692;
6693; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_load:
6694; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6695; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6696; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6697; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6698; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6699; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6700; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6701; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6702; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6703; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6704; GFX90A-TGSPLIT-NEXT:    s_endpgm
6705;
6706; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_load:
6707; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6708; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6709; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6710; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6711; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6712; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6713; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6714; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6715; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6716; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6717; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6718;
6719; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_load:
6720; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6721; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6722; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6723; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6724; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6725; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6726; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6727; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6728; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6729; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6730; GFX940-TGSPLIT-NEXT:    s_endpgm
6731;
6732; GFX11-WGP-LABEL: flat_singlethread_one_as_monotonic_load:
6733; GFX11-WGP:       ; %bb.0: ; %entry
6734; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6735; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6736; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
6737; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
6738; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
6739; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6740; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6741; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6742; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6743; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6744; GFX11-WGP-NEXT:    s_endpgm
6745;
6746; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_load:
6747; GFX11-CU:       ; %bb.0: ; %entry
6748; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6749; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6750; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
6751; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
6752; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
6753; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6754; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6755; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6756; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6757; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6758; GFX11-CU-NEXT:    s_endpgm
6759    i32* %in, i32* %out) {
6760entry:
6761  %val = load atomic i32, i32* %in syncscope("singlethread-one-as") monotonic, align 4
6762  store i32 %val, i32* %out
6763  ret void
6764}
6765
6766define amdgpu_kernel void @flat_singlethread_one_as_acquire_load(
6767; GFX7-LABEL: flat_singlethread_one_as_acquire_load:
6768; GFX7:       ; %bb.0: ; %entry
6769; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6770; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6771; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6772; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6773; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6774; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6775; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6776; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6777; GFX7-NEXT:    flat_store_dword v[0:1], v2
6778; GFX7-NEXT:    s_endpgm
6779;
6780; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_load:
6781; GFX10-WGP:       ; %bb.0: ; %entry
6782; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6783; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6784; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6785; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6786; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
6787; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6788; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6789; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6790; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6791; GFX10-WGP-NEXT:    s_endpgm
6792;
6793; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_load:
6794; GFX10-CU:       ; %bb.0: ; %entry
6795; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6796; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6797; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6798; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6799; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6800; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6801; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6802; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6803; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6804; GFX10-CU-NEXT:    s_endpgm
6805;
6806; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_load:
6807; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6808; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6809; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6810; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6811; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6812; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
6813; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6814; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6815; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6816; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6817; SKIP-CACHE-INV-NEXT:    s_endpgm
6818;
6819; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_load:
6820; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6821; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6822; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6823; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6824; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6825; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6826; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6827; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6828; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6829; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6830; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6831;
6832; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_load:
6833; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6834; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6835; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6836; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6837; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6838; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6839; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6840; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6841; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6842; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6843; GFX90A-TGSPLIT-NEXT:    s_endpgm
6844;
6845; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_load:
6846; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6847; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6848; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6849; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6850; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6851; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6852; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6853; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6854; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6855; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6856; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6857;
6858; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_load:
6859; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6860; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6861; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6862; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6863; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6864; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6865; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6866; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6867; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6868; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6869; GFX940-TGSPLIT-NEXT:    s_endpgm
6870;
6871; GFX11-WGP-LABEL: flat_singlethread_one_as_acquire_load:
6872; GFX11-WGP:       ; %bb.0: ; %entry
6873; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6874; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6875; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
6876; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
6877; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
6878; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6879; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6880; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6881; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6882; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6883; GFX11-WGP-NEXT:    s_endpgm
6884;
6885; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_load:
6886; GFX11-CU:       ; %bb.0: ; %entry
6887; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6888; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6889; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
6890; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
6891; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
6892; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6893; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6894; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6895; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6896; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6897; GFX11-CU-NEXT:    s_endpgm
6898    i32* %in, i32* %out) {
6899entry:
6900  %val = load atomic i32, i32* %in syncscope("singlethread-one-as") acquire, align 4
6901  store i32 %val, i32* %out
6902  ret void
6903}
6904
6905define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_load(
6906; GFX7-LABEL: flat_singlethread_one_as_seq_cst_load:
6907; GFX7:       ; %bb.0: ; %entry
6908; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6909; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6910; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6911; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6912; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6913; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6914; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6915; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6916; GFX7-NEXT:    flat_store_dword v[0:1], v2
6917; GFX7-NEXT:    s_endpgm
6918;
6919; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_load:
6920; GFX10-WGP:       ; %bb.0: ; %entry
6921; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6922; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6923; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6924; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6925; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
6926; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6927; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6928; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6929; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6930; GFX10-WGP-NEXT:    s_endpgm
6931;
6932; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_load:
6933; GFX10-CU:       ; %bb.0: ; %entry
6934; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6935; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6936; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6937; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6938; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6939; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6940; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6941; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6942; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6943; GFX10-CU-NEXT:    s_endpgm
6944;
6945; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_load:
6946; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6947; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6948; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6949; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6950; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6951; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
6952; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6953; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6954; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6955; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6956; SKIP-CACHE-INV-NEXT:    s_endpgm
6957;
6958; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_load:
6959; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6960; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6961; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6962; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6963; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6964; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6965; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6966; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6967; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6968; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6969; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6970;
6971; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_load:
6972; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6973; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6974; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6975; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6976; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6977; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6978; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6979; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6980; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6981; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6982; GFX90A-TGSPLIT-NEXT:    s_endpgm
6983;
6984; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_load:
6985; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6986; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6987; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6988; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6989; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6990; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6991; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6992; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6993; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6994; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6995; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6996;
6997; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_load:
6998; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6999; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7000; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7001; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7002; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7003; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
7004; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7005; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7006; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7007; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7008; GFX940-TGSPLIT-NEXT:    s_endpgm
7009;
7010; GFX11-WGP-LABEL: flat_singlethread_one_as_seq_cst_load:
7011; GFX11-WGP:       ; %bb.0: ; %entry
7012; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7013; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7014; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
7015; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
7016; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
7017; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7018; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7019; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7020; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7021; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7022; GFX11-WGP-NEXT:    s_endpgm
7023;
7024; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_load:
7025; GFX11-CU:       ; %bb.0: ; %entry
7026; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7027; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7028; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
7029; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
7030; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
7031; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7032; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7033; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7034; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7035; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7036; GFX11-CU-NEXT:    s_endpgm
7037    i32* %in, i32* %out) {
7038entry:
7039  %val = load atomic i32, i32* %in syncscope("singlethread-one-as") seq_cst, align 4
7040  store i32 %val, i32* %out
7041  ret void
7042}
7043
7044define amdgpu_kernel void @flat_singlethread_one_as_unordered_store(
7045; GFX7-LABEL: flat_singlethread_one_as_unordered_store:
7046; GFX7:       ; %bb.0: ; %entry
7047; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7048; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7049; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7050; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7051; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7052; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7053; GFX7-NEXT:    flat_store_dword v[0:1], v2
7054; GFX7-NEXT:    s_endpgm
7055;
7056; GFX10-WGP-LABEL: flat_singlethread_one_as_unordered_store:
7057; GFX10-WGP:       ; %bb.0: ; %entry
7058; GFX10-WGP-NEXT:    s_clause 0x1
7059; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7060; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7061; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7062; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7063; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7064; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7065; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7066; GFX10-WGP-NEXT:    s_endpgm
7067;
7068; GFX10-CU-LABEL: flat_singlethread_one_as_unordered_store:
7069; GFX10-CU:       ; %bb.0: ; %entry
7070; GFX10-CU-NEXT:    s_clause 0x1
7071; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7072; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7073; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7074; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7075; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7076; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7077; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7078; GFX10-CU-NEXT:    s_endpgm
7079;
7080; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_unordered_store:
7081; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7082; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
7083; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
7084; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7085; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7086; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7087; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7088; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7089; SKIP-CACHE-INV-NEXT:    s_endpgm
7090;
7091; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_unordered_store:
7092; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7093; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7094; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7095; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7096; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7097; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7098; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7099; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7100;
7101; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_unordered_store:
7102; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7103; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7104; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7105; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7106; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7107; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7108; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7109; GFX90A-TGSPLIT-NEXT:    s_endpgm
7110;
7111; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_unordered_store:
7112; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7113; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7114; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7115; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7116; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7117; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7118; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7119; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7120;
7121; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_unordered_store:
7122; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7123; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7124; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7125; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7126; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7127; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7128; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7129; GFX940-TGSPLIT-NEXT:    s_endpgm
7130;
7131; GFX11-WGP-LABEL: flat_singlethread_one_as_unordered_store:
7132; GFX11-WGP:       ; %bb.0: ; %entry
7133; GFX11-WGP-NEXT:    s_clause 0x1
7134; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7135; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
7136; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7137; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7138; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7139; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7140; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7141; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7142; GFX11-WGP-NEXT:    s_endpgm
7143;
7144; GFX11-CU-LABEL: flat_singlethread_one_as_unordered_store:
7145; GFX11-CU:       ; %bb.0: ; %entry
7146; GFX11-CU-NEXT:    s_clause 0x1
7147; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7148; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
7149; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7150; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7151; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7152; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7153; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7154; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7155; GFX11-CU-NEXT:    s_endpgm
7156    i32 %in, i32* %out) {
7157entry:
7158  store atomic i32 %in, i32* %out syncscope("singlethread-one-as") unordered, align 4
7159  ret void
7160}
7161
7162define amdgpu_kernel void @flat_singlethread_one_as_monotonic_store(
7163; GFX7-LABEL: flat_singlethread_one_as_monotonic_store:
7164; GFX7:       ; %bb.0: ; %entry
7165; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7166; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7167; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7168; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7169; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7170; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7171; GFX7-NEXT:    flat_store_dword v[0:1], v2
7172; GFX7-NEXT:    s_endpgm
7173;
7174; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_store:
7175; GFX10-WGP:       ; %bb.0: ; %entry
7176; GFX10-WGP-NEXT:    s_clause 0x1
7177; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7178; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7179; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7180; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7181; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7182; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7183; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7184; GFX10-WGP-NEXT:    s_endpgm
7185;
7186; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_store:
7187; GFX10-CU:       ; %bb.0: ; %entry
7188; GFX10-CU-NEXT:    s_clause 0x1
7189; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7190; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7191; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7192; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7193; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7194; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7195; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7196; GFX10-CU-NEXT:    s_endpgm
7197;
7198; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_store:
7199; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7200; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
7201; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
7202; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7203; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7204; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7205; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7206; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7207; SKIP-CACHE-INV-NEXT:    s_endpgm
7208;
7209; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_store:
7210; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7211; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7212; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7213; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7214; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7215; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7216; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7217; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7218;
7219; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_store:
7220; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7221; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7222; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7223; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7224; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7225; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7226; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7227; GFX90A-TGSPLIT-NEXT:    s_endpgm
7228;
7229; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_store:
7230; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7231; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7232; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7233; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7234; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7235; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7236; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7237; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7238;
7239; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_store:
7240; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7241; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7242; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7243; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7244; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7245; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7246; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7247; GFX940-TGSPLIT-NEXT:    s_endpgm
7248;
7249; GFX11-WGP-LABEL: flat_singlethread_one_as_monotonic_store:
7250; GFX11-WGP:       ; %bb.0: ; %entry
7251; GFX11-WGP-NEXT:    s_clause 0x1
7252; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7253; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
7254; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7255; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7256; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7257; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7258; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7259; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7260; GFX11-WGP-NEXT:    s_endpgm
7261;
7262; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_store:
7263; GFX11-CU:       ; %bb.0: ; %entry
7264; GFX11-CU-NEXT:    s_clause 0x1
7265; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7266; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
7267; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7268; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7269; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7270; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7271; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7272; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7273; GFX11-CU-NEXT:    s_endpgm
7274    i32 %in, i32* %out) {
7275entry:
7276  store atomic i32 %in, i32* %out syncscope("singlethread-one-as") monotonic, align 4
7277  ret void
7278}
7279
7280define amdgpu_kernel void @flat_singlethread_one_as_release_store(
7281; GFX7-LABEL: flat_singlethread_one_as_release_store:
7282; GFX7:       ; %bb.0: ; %entry
7283; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7284; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7285; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7286; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7287; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7288; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7289; GFX7-NEXT:    flat_store_dword v[0:1], v2
7290; GFX7-NEXT:    s_endpgm
7291;
7292; GFX10-WGP-LABEL: flat_singlethread_one_as_release_store:
7293; GFX10-WGP:       ; %bb.0: ; %entry
7294; GFX10-WGP-NEXT:    s_clause 0x1
7295; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7296; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7297; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7298; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7299; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7300; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7301; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7302; GFX10-WGP-NEXT:    s_endpgm
7303;
7304; GFX10-CU-LABEL: flat_singlethread_one_as_release_store:
7305; GFX10-CU:       ; %bb.0: ; %entry
7306; GFX10-CU-NEXT:    s_clause 0x1
7307; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7308; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7309; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7310; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7311; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7312; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7313; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7314; GFX10-CU-NEXT:    s_endpgm
7315;
7316; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_store:
7317; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7318; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
7319; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
7320; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7321; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7322; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7323; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7324; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7325; SKIP-CACHE-INV-NEXT:    s_endpgm
7326;
7327; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_store:
7328; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7329; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7330; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7331; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7332; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7333; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7334; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7335; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7336;
7337; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_store:
7338; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7339; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7340; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7341; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7342; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7343; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7344; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7345; GFX90A-TGSPLIT-NEXT:    s_endpgm
7346;
7347; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_store:
7348; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7349; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7350; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7351; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7352; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7353; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7354; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7355; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7356;
7357; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_store:
7358; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7359; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7360; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7361; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7362; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7363; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7364; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7365; GFX940-TGSPLIT-NEXT:    s_endpgm
7366;
7367; GFX11-WGP-LABEL: flat_singlethread_one_as_release_store:
7368; GFX11-WGP:       ; %bb.0: ; %entry
7369; GFX11-WGP-NEXT:    s_clause 0x1
7370; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7371; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
7372; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7373; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7374; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7375; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7376; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7377; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7378; GFX11-WGP-NEXT:    s_endpgm
7379;
7380; GFX11-CU-LABEL: flat_singlethread_one_as_release_store:
7381; GFX11-CU:       ; %bb.0: ; %entry
7382; GFX11-CU-NEXT:    s_clause 0x1
7383; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7384; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
7385; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7386; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7387; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7388; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7389; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7390; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7391; GFX11-CU-NEXT:    s_endpgm
7392    i32 %in, i32* %out) {
7393entry:
7394  store atomic i32 %in, i32* %out syncscope("singlethread-one-as") release, align 4
7395  ret void
7396}
7397
7398define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_store(
7399; GFX7-LABEL: flat_singlethread_one_as_seq_cst_store:
7400; GFX7:       ; %bb.0: ; %entry
7401; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7402; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7403; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7404; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7405; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7406; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7407; GFX7-NEXT:    flat_store_dword v[0:1], v2
7408; GFX7-NEXT:    s_endpgm
7409;
7410; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_store:
7411; GFX10-WGP:       ; %bb.0: ; %entry
7412; GFX10-WGP-NEXT:    s_clause 0x1
7413; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7414; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7415; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7416; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7417; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7418; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7419; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7420; GFX10-WGP-NEXT:    s_endpgm
7421;
7422; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_store:
7423; GFX10-CU:       ; %bb.0: ; %entry
7424; GFX10-CU-NEXT:    s_clause 0x1
7425; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7426; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7427; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7428; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7429; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7430; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7431; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7432; GFX10-CU-NEXT:    s_endpgm
7433;
7434; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_store:
7435; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7436; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
7437; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
7438; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7439; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7440; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7441; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7442; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7443; SKIP-CACHE-INV-NEXT:    s_endpgm
7444;
7445; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_store:
7446; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7447; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7448; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7449; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7450; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7451; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7452; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7453; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7454;
7455; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_store:
7456; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7457; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7458; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7459; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7460; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7461; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7462; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7463; GFX90A-TGSPLIT-NEXT:    s_endpgm
7464;
7465; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_store:
7466; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7467; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7468; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7469; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7470; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7471; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7472; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7473; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7474;
7475; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_store:
7476; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7477; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7478; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7479; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7480; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7481; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7482; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7483; GFX940-TGSPLIT-NEXT:    s_endpgm
7484;
7485; GFX11-WGP-LABEL: flat_singlethread_one_as_seq_cst_store:
7486; GFX11-WGP:       ; %bb.0: ; %entry
7487; GFX11-WGP-NEXT:    s_clause 0x1
7488; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7489; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
7490; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7491; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7492; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7493; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7494; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7495; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7496; GFX11-WGP-NEXT:    s_endpgm
7497;
7498; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_store:
7499; GFX11-CU:       ; %bb.0: ; %entry
7500; GFX11-CU-NEXT:    s_clause 0x1
7501; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7502; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
7503; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7504; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7505; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7506; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7507; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7508; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7509; GFX11-CU-NEXT:    s_endpgm
7510    i32 %in, i32* %out) {
7511entry:
7512  store atomic i32 %in, i32* %out syncscope("singlethread-one-as") seq_cst, align 4
7513  ret void
7514}
7515
7516define amdgpu_kernel void @flat_singlethread_one_as_monotonic_atomicrmw(
7517; GFX7-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
7518; GFX7:       ; %bb.0: ; %entry
7519; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7520; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7521; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7522; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7523; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7524; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7525; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7526; GFX7-NEXT:    s_endpgm
7527;
7528; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
7529; GFX10-WGP:       ; %bb.0: ; %entry
7530; GFX10-WGP-NEXT:    s_clause 0x1
7531; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7532; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7533; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7534; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7535; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7536; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7537; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7538; GFX10-WGP-NEXT:    s_endpgm
7539;
7540; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
7541; GFX10-CU:       ; %bb.0: ; %entry
7542; GFX10-CU-NEXT:    s_clause 0x1
7543; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7544; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7545; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7546; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7547; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7548; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7549; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7550; GFX10-CU-NEXT:    s_endpgm
7551;
7552; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
7553; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7554; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7555; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7556; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7557; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7558; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7559; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7560; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7561; SKIP-CACHE-INV-NEXT:    s_endpgm
7562;
7563; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
7564; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7565; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7566; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7567; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7568; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7569; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7570; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7571; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7572;
7573; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
7574; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7575; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7576; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7577; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7578; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7579; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7580; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7581; GFX90A-TGSPLIT-NEXT:    s_endpgm
7582;
7583; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
7584; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7585; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7586; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7587; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7588; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7589; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7590; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7591; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7592;
7593; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
7594; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7595; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7596; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7597; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7598; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7599; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7600; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7601; GFX940-TGSPLIT-NEXT:    s_endpgm
7602;
7603; GFX11-WGP-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
7604; GFX11-WGP:       ; %bb.0: ; %entry
7605; GFX11-WGP-NEXT:    s_clause 0x1
7606; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7607; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
7608; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7609; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7610; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7611; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7612; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7613; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7614; GFX11-WGP-NEXT:    s_endpgm
7615;
7616; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
7617; GFX11-CU:       ; %bb.0: ; %entry
7618; GFX11-CU-NEXT:    s_clause 0x1
7619; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7620; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7621; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7622; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7623; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7624; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7625; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7626; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7627; GFX11-CU-NEXT:    s_endpgm
7628    i32* %out, i32 %in) {
7629entry:
7630  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") monotonic
7631  ret void
7632}
7633
7634define amdgpu_kernel void @flat_singlethread_one_as_acquire_atomicrmw(
7635; GFX7-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
7636; GFX7:       ; %bb.0: ; %entry
7637; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7638; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7639; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7640; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7641; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7642; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7643; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7644; GFX7-NEXT:    s_endpgm
7645;
7646; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
7647; GFX10-WGP:       ; %bb.0: ; %entry
7648; GFX10-WGP-NEXT:    s_clause 0x1
7649; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7650; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7651; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7652; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7653; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7654; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7655; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7656; GFX10-WGP-NEXT:    s_endpgm
7657;
7658; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
7659; GFX10-CU:       ; %bb.0: ; %entry
7660; GFX10-CU-NEXT:    s_clause 0x1
7661; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7662; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7663; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7664; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7665; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7666; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7667; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7668; GFX10-CU-NEXT:    s_endpgm
7669;
7670; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
7671; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7672; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7673; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7674; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7675; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7676; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7677; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7678; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7679; SKIP-CACHE-INV-NEXT:    s_endpgm
7680;
7681; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
7682; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7683; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7684; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7685; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7686; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7687; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7688; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7689; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7690;
7691; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
7692; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7693; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7694; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7695; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7696; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7697; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7698; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7699; GFX90A-TGSPLIT-NEXT:    s_endpgm
7700;
7701; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
7702; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7703; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7704; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7705; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7706; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7707; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7708; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7709; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7710;
7711; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
7712; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7713; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7714; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7715; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7716; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7717; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7718; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7719; GFX940-TGSPLIT-NEXT:    s_endpgm
7720;
7721; GFX11-WGP-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
7722; GFX11-WGP:       ; %bb.0: ; %entry
7723; GFX11-WGP-NEXT:    s_clause 0x1
7724; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7725; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
7726; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7727; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7728; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7729; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7730; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7731; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7732; GFX11-WGP-NEXT:    s_endpgm
7733;
7734; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
7735; GFX11-CU:       ; %bb.0: ; %entry
7736; GFX11-CU-NEXT:    s_clause 0x1
7737; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7738; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7739; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7740; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7741; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7742; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7743; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7744; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7745; GFX11-CU-NEXT:    s_endpgm
7746    i32* %out, i32 %in) {
7747entry:
7748  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") acquire
7749  ret void
7750}
7751
7752define amdgpu_kernel void @flat_singlethread_one_as_release_atomicrmw(
7753; GFX7-LABEL: flat_singlethread_one_as_release_atomicrmw:
7754; GFX7:       ; %bb.0: ; %entry
7755; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7756; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7757; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7758; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7759; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7760; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7761; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7762; GFX7-NEXT:    s_endpgm
7763;
7764; GFX10-WGP-LABEL: flat_singlethread_one_as_release_atomicrmw:
7765; GFX10-WGP:       ; %bb.0: ; %entry
7766; GFX10-WGP-NEXT:    s_clause 0x1
7767; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7768; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7769; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7770; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7771; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7772; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7773; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7774; GFX10-WGP-NEXT:    s_endpgm
7775;
7776; GFX10-CU-LABEL: flat_singlethread_one_as_release_atomicrmw:
7777; GFX10-CU:       ; %bb.0: ; %entry
7778; GFX10-CU-NEXT:    s_clause 0x1
7779; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7780; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7781; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7782; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7783; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7784; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7785; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7786; GFX10-CU-NEXT:    s_endpgm
7787;
7788; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_atomicrmw:
7789; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7790; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7791; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7792; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7793; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7794; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7795; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7796; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7797; SKIP-CACHE-INV-NEXT:    s_endpgm
7798;
7799; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_atomicrmw:
7800; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7801; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7802; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7803; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7804; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7805; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7806; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7807; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7808;
7809; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_atomicrmw:
7810; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7811; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7812; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7813; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7814; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7815; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7816; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7817; GFX90A-TGSPLIT-NEXT:    s_endpgm
7818;
7819; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_atomicrmw:
7820; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7821; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7822; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7823; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7824; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7825; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7826; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7827; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7828;
7829; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_atomicrmw:
7830; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7831; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7832; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7833; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7834; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7835; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7836; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7837; GFX940-TGSPLIT-NEXT:    s_endpgm
7838;
7839; GFX11-WGP-LABEL: flat_singlethread_one_as_release_atomicrmw:
7840; GFX11-WGP:       ; %bb.0: ; %entry
7841; GFX11-WGP-NEXT:    s_clause 0x1
7842; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7843; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
7844; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7845; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7846; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7847; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7848; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7849; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7850; GFX11-WGP-NEXT:    s_endpgm
7851;
7852; GFX11-CU-LABEL: flat_singlethread_one_as_release_atomicrmw:
7853; GFX11-CU:       ; %bb.0: ; %entry
7854; GFX11-CU-NEXT:    s_clause 0x1
7855; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7856; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7857; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7858; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7859; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7860; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7861; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7862; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7863; GFX11-CU-NEXT:    s_endpgm
7864    i32* %out, i32 %in) {
7865entry:
7866  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") release
7867  ret void
7868}
7869
7870define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_atomicrmw(
7871; GFX7-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
7872; GFX7:       ; %bb.0: ; %entry
7873; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7874; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7875; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7876; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7877; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7878; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7879; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7880; GFX7-NEXT:    s_endpgm
7881;
7882; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
7883; GFX10-WGP:       ; %bb.0: ; %entry
7884; GFX10-WGP-NEXT:    s_clause 0x1
7885; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7886; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7887; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7888; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7889; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7890; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7891; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7892; GFX10-WGP-NEXT:    s_endpgm
7893;
7894; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
7895; GFX10-CU:       ; %bb.0: ; %entry
7896; GFX10-CU-NEXT:    s_clause 0x1
7897; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7898; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7899; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7900; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7901; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7902; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7903; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7904; GFX10-CU-NEXT:    s_endpgm
7905;
7906; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
7907; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7908; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7909; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7910; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7911; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7912; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7913; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7914; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7915; SKIP-CACHE-INV-NEXT:    s_endpgm
7916;
7917; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
7918; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7919; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7920; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7921; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7922; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7923; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7924; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7925; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7926;
7927; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
7928; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7929; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7930; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7931; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7932; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7933; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7934; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7935; GFX90A-TGSPLIT-NEXT:    s_endpgm
7936;
7937; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
7938; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7939; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7940; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7941; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7942; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7943; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7944; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7945; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7946;
7947; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
7948; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7949; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7950; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7951; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7952; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7953; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7954; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7955; GFX940-TGSPLIT-NEXT:    s_endpgm
7956;
7957; GFX11-WGP-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
7958; GFX11-WGP:       ; %bb.0: ; %entry
7959; GFX11-WGP-NEXT:    s_clause 0x1
7960; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7961; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
7962; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7963; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7964; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7965; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7966; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7967; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7968; GFX11-WGP-NEXT:    s_endpgm
7969;
7970; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
7971; GFX11-CU:       ; %bb.0: ; %entry
7972; GFX11-CU-NEXT:    s_clause 0x1
7973; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7974; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7975; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7976; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7977; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7978; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7979; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7980; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7981; GFX11-CU-NEXT:    s_endpgm
7982    i32* %out, i32 %in) {
7983entry:
7984  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") acq_rel
7985  ret void
7986}
7987
7988define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_atomicrmw(
7989; GFX7-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
7990; GFX7:       ; %bb.0: ; %entry
7991; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7992; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7993; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7994; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7995; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7996; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7997; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7998; GFX7-NEXT:    s_endpgm
7999;
8000; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
8001; GFX10-WGP:       ; %bb.0: ; %entry
8002; GFX10-WGP-NEXT:    s_clause 0x1
8003; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8004; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8005; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8006; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8007; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8008; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8009; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
8010; GFX10-WGP-NEXT:    s_endpgm
8011;
8012; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
8013; GFX10-CU:       ; %bb.0: ; %entry
8014; GFX10-CU-NEXT:    s_clause 0x1
8015; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8016; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8017; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8018; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8019; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8020; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8021; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
8022; GFX10-CU-NEXT:    s_endpgm
8023;
8024; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
8025; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8026; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8027; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8028; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8029; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8030; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8031; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8032; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
8033; SKIP-CACHE-INV-NEXT:    s_endpgm
8034;
8035; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
8036; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8037; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8038; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8039; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8040; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8041; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8042; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8043; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8044;
8045; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
8046; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8047; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8048; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8049; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8050; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8051; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8052; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8053; GFX90A-TGSPLIT-NEXT:    s_endpgm
8054;
8055; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
8056; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8057; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8058; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8059; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8060; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8061; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8062; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8063; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8064;
8065; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
8066; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8067; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8068; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8069; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8070; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8071; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8072; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8073; GFX940-TGSPLIT-NEXT:    s_endpgm
8074;
8075; GFX11-WGP-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
8076; GFX11-WGP:       ; %bb.0: ; %entry
8077; GFX11-WGP-NEXT:    s_clause 0x1
8078; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8079; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8080; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8081; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8082; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8083; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8084; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8085; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8086; GFX11-WGP-NEXT:    s_endpgm
8087;
8088; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
8089; GFX11-CU:       ; %bb.0: ; %entry
8090; GFX11-CU-NEXT:    s_clause 0x1
8091; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8092; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8093; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8094; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8095; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8096; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8097; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8098; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8099; GFX11-CU-NEXT:    s_endpgm
8100    i32* %out, i32 %in) {
8101entry:
8102  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") seq_cst
8103  ret void
8104}
8105
8106define amdgpu_kernel void @flat_singlethread_one_as_acquire_ret_atomicrmw(
8107; GFX7-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
8108; GFX7:       ; %bb.0: ; %entry
8109; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8110; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8111; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8112; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8113; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8114; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8115; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8116; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8117; GFX7-NEXT:    flat_store_dword v[0:1], v2
8118; GFX7-NEXT:    s_endpgm
8119;
8120; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
8121; GFX10-WGP:       ; %bb.0: ; %entry
8122; GFX10-WGP-NEXT:    s_clause 0x1
8123; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8124; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8125; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8126; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8127; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8128; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8129; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8130; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8131; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8132; GFX10-WGP-NEXT:    s_endpgm
8133;
8134; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
8135; GFX10-CU:       ; %bb.0: ; %entry
8136; GFX10-CU-NEXT:    s_clause 0x1
8137; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8138; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8139; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8140; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8141; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8142; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8143; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8144; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8145; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8146; GFX10-CU-NEXT:    s_endpgm
8147;
8148; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
8149; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8150; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8151; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8152; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8153; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8154; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8155; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8156; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8157; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8158; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8159; SKIP-CACHE-INV-NEXT:    s_endpgm
8160;
8161; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
8162; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8163; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8164; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8165; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8166; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8167; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8168; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8169; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8170; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8171; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8172;
8173; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
8174; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8175; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8176; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8177; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8178; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8179; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8180; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8181; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8182; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8183; GFX90A-TGSPLIT-NEXT:    s_endpgm
8184;
8185; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
8186; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8187; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8188; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8189; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8190; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8191; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8192; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8193; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8194; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8195; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8196;
8197; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
8198; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8199; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8200; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8201; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8202; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8203; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8204; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8205; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8206; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8207; GFX940-TGSPLIT-NEXT:    s_endpgm
8208;
8209; GFX11-WGP-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
8210; GFX11-WGP:       ; %bb.0: ; %entry
8211; GFX11-WGP-NEXT:    s_clause 0x1
8212; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8213; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8214; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8215; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8216; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8217; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8218; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8219; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8220; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8221; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8222; GFX11-WGP-NEXT:    s_endpgm
8223;
8224; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
8225; GFX11-CU:       ; %bb.0: ; %entry
8226; GFX11-CU-NEXT:    s_clause 0x1
8227; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8228; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8229; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8230; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8231; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8232; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8233; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8234; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8235; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8236; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8237; GFX11-CU-NEXT:    s_endpgm
8238    i32* %out, i32 %in) {
8239entry:
8240  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") acquire
8241  store i32 %val, i32* %out, align 4
8242  ret void
8243}
8244
8245define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_ret_atomicrmw(
8246; GFX7-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
8247; GFX7:       ; %bb.0: ; %entry
8248; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8249; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8250; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8251; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8252; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8253; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8254; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8255; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8256; GFX7-NEXT:    flat_store_dword v[0:1], v2
8257; GFX7-NEXT:    s_endpgm
8258;
8259; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
8260; GFX10-WGP:       ; %bb.0: ; %entry
8261; GFX10-WGP-NEXT:    s_clause 0x1
8262; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8263; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8264; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8265; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8266; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8267; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8268; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8269; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8270; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8271; GFX10-WGP-NEXT:    s_endpgm
8272;
8273; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
8274; GFX10-CU:       ; %bb.0: ; %entry
8275; GFX10-CU-NEXT:    s_clause 0x1
8276; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8277; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8278; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8279; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8280; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8281; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8282; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8283; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8284; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8285; GFX10-CU-NEXT:    s_endpgm
8286;
8287; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
8288; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8289; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8290; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8291; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8292; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8293; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8294; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8295; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8296; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8297; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8298; SKIP-CACHE-INV-NEXT:    s_endpgm
8299;
8300; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
8301; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8302; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8303; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8304; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8305; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8306; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8307; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8308; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8309; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8310; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8311;
8312; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
8313; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8314; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8315; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8316; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8317; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8318; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8319; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8320; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8321; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8322; GFX90A-TGSPLIT-NEXT:    s_endpgm
8323;
8324; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
8325; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8326; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8327; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8328; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8329; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8330; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8331; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8332; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8333; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8334; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8335;
8336; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
8337; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8338; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8339; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8340; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8341; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8342; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8343; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8344; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8345; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8346; GFX940-TGSPLIT-NEXT:    s_endpgm
8347;
8348; GFX11-WGP-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
8349; GFX11-WGP:       ; %bb.0: ; %entry
8350; GFX11-WGP-NEXT:    s_clause 0x1
8351; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8352; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8353; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8354; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8355; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8356; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8357; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8358; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8359; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8360; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8361; GFX11-WGP-NEXT:    s_endpgm
8362;
8363; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
8364; GFX11-CU:       ; %bb.0: ; %entry
8365; GFX11-CU-NEXT:    s_clause 0x1
8366; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8367; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8368; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8369; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8370; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8371; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8372; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8373; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8374; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8375; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8376; GFX11-CU-NEXT:    s_endpgm
8377    i32* %out, i32 %in) {
8378entry:
8379  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") acq_rel
8380  store i32 %val, i32* %out, align 4
8381  ret void
8382}
8383
8384define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_ret_atomicrmw(
8385; GFX7-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
8386; GFX7:       ; %bb.0: ; %entry
8387; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8388; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8389; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8390; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8391; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8392; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8393; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8394; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8395; GFX7-NEXT:    flat_store_dword v[0:1], v2
8396; GFX7-NEXT:    s_endpgm
8397;
8398; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
8399; GFX10-WGP:       ; %bb.0: ; %entry
8400; GFX10-WGP-NEXT:    s_clause 0x1
8401; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8402; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8403; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8404; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8405; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8406; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8407; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8408; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8409; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8410; GFX10-WGP-NEXT:    s_endpgm
8411;
8412; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
8413; GFX10-CU:       ; %bb.0: ; %entry
8414; GFX10-CU-NEXT:    s_clause 0x1
8415; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8416; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8417; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8418; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8419; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8420; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8421; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8422; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8423; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8424; GFX10-CU-NEXT:    s_endpgm
8425;
8426; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
8427; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8428; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8429; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8430; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8431; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8432; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8433; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8434; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8435; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8436; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8437; SKIP-CACHE-INV-NEXT:    s_endpgm
8438;
8439; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
8440; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8441; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8442; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8443; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8444; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8445; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8446; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8447; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8448; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8449; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8450;
8451; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
8452; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8453; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8454; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8455; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8456; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8457; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8458; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8459; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8460; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8461; GFX90A-TGSPLIT-NEXT:    s_endpgm
8462;
8463; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
8464; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8465; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8466; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8467; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8468; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8469; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8470; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8471; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8472; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8473; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8474;
8475; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
8476; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8477; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8478; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8479; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8480; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8481; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8482; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8483; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8484; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8485; GFX940-TGSPLIT-NEXT:    s_endpgm
8486;
8487; GFX11-WGP-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
8488; GFX11-WGP:       ; %bb.0: ; %entry
8489; GFX11-WGP-NEXT:    s_clause 0x1
8490; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8491; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8492; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8493; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8494; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8495; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8496; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8497; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8498; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8499; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8500; GFX11-WGP-NEXT:    s_endpgm
8501;
8502; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
8503; GFX11-CU:       ; %bb.0: ; %entry
8504; GFX11-CU-NEXT:    s_clause 0x1
8505; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8506; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8507; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8508; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8509; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8510; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8511; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8512; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8513; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8514; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8515; GFX11-CU-NEXT:    s_endpgm
8516    i32* %out, i32 %in) {
8517entry:
8518  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") seq_cst
8519  store i32 %val, i32* %out, align 4
8520  ret void
8521}
8522
8523define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_cmpxchg(
8524; GFX7-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
8525; GFX7:       ; %bb.0: ; %entry
8526; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8527; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8528; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8529; GFX7-NEXT:    s_add_u32 s0, s0, 16
8530; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8531; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8532; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8533; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8534; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8535; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8536; GFX7-NEXT:    s_endpgm
8537;
8538; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
8539; GFX10-WGP:       ; %bb.0: ; %entry
8540; GFX10-WGP-NEXT:    s_clause 0x1
8541; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8542; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8543; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8544; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8545; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8546; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8547; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8548; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8549; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8550; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8551; GFX10-WGP-NEXT:    s_endpgm
8552;
8553; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
8554; GFX10-CU:       ; %bb.0: ; %entry
8555; GFX10-CU-NEXT:    s_clause 0x1
8556; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8557; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8558; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8559; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8560; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8561; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8562; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8563; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8564; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8565; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8566; GFX10-CU-NEXT:    s_endpgm
8567;
8568; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
8569; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8570; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8571; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8572; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8573; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8574; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8575; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8576; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8577; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8578; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8579; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8580; SKIP-CACHE-INV-NEXT:    s_endpgm
8581;
8582; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
8583; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8584; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8585; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8586; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8587; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8588; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8589; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8590; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8591;
8592; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
8593; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8594; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8595; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8596; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8597; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8598; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8599; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8600; GFX90A-TGSPLIT-NEXT:    s_endpgm
8601;
8602; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
8603; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8604; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8605; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8606; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8607; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8608; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8609; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8610; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8611;
8612; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
8613; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8614; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8615; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8616; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8617; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8618; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8619; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8620; GFX940-TGSPLIT-NEXT:    s_endpgm
8621;
8622; GFX11-WGP-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
8623; GFX11-WGP:       ; %bb.0: ; %entry
8624; GFX11-WGP-NEXT:    s_clause 0x1
8625; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8626; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8627; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8628; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8629; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
8630; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8631; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8632; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8633; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8634; GFX11-WGP-NEXT:    s_endpgm
8635;
8636; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
8637; GFX11-CU:       ; %bb.0: ; %entry
8638; GFX11-CU-NEXT:    s_clause 0x1
8639; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8640; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8641; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8642; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8643; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
8644; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8645; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8646; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8647; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8648; GFX11-CU-NEXT:    s_endpgm
8649    i32* %out, i32 %in, i32 %old) {
8650entry:
8651  %gep = getelementptr i32, i32* %out, i32 4
8652  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
8653  ret void
8654}
8655
8656define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_cmpxchg(
8657; GFX7-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
8658; GFX7:       ; %bb.0: ; %entry
8659; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8660; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8661; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8662; GFX7-NEXT:    s_add_u32 s0, s0, 16
8663; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8664; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8665; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8666; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8667; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8668; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8669; GFX7-NEXT:    s_endpgm
8670;
8671; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
8672; GFX10-WGP:       ; %bb.0: ; %entry
8673; GFX10-WGP-NEXT:    s_clause 0x1
8674; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8675; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8676; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8677; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8678; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8679; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8680; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8681; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8682; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8683; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8684; GFX10-WGP-NEXT:    s_endpgm
8685;
8686; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
8687; GFX10-CU:       ; %bb.0: ; %entry
8688; GFX10-CU-NEXT:    s_clause 0x1
8689; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8690; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8691; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8692; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8693; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8694; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8695; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8696; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8697; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8698; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8699; GFX10-CU-NEXT:    s_endpgm
8700;
8701; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
8702; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8703; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8704; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8705; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8706; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8707; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8708; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8709; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8710; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8711; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8712; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8713; SKIP-CACHE-INV-NEXT:    s_endpgm
8714;
8715; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
8716; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8717; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8718; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8719; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8720; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8721; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8722; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8723; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8724;
8725; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
8726; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8727; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8728; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8729; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8730; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8731; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8732; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8733; GFX90A-TGSPLIT-NEXT:    s_endpgm
8734;
8735; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
8736; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8737; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8738; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8739; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8740; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8741; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8742; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8743; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8744;
8745; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
8746; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8747; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8748; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8749; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8750; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8751; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8752; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8753; GFX940-TGSPLIT-NEXT:    s_endpgm
8754;
8755; GFX11-WGP-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
8756; GFX11-WGP:       ; %bb.0: ; %entry
8757; GFX11-WGP-NEXT:    s_clause 0x1
8758; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8759; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8760; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8761; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8762; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
8763; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8764; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8765; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8766; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8767; GFX11-WGP-NEXT:    s_endpgm
8768;
8769; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
8770; GFX11-CU:       ; %bb.0: ; %entry
8771; GFX11-CU-NEXT:    s_clause 0x1
8772; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8773; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8774; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8775; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8776; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
8777; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8778; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8779; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8780; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8781; GFX11-CU-NEXT:    s_endpgm
8782    i32* %out, i32 %in, i32 %old) {
8783entry:
8784  %gep = getelementptr i32, i32* %out, i32 4
8785  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
8786  ret void
8787}
8788
8789define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_cmpxchg(
8790; GFX7-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
8791; GFX7:       ; %bb.0: ; %entry
8792; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8793; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8794; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8795; GFX7-NEXT:    s_add_u32 s0, s0, 16
8796; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8797; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8798; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8799; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8800; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8801; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8802; GFX7-NEXT:    s_endpgm
8803;
8804; GFX10-WGP-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
8805; GFX10-WGP:       ; %bb.0: ; %entry
8806; GFX10-WGP-NEXT:    s_clause 0x1
8807; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8808; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8809; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8810; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8811; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8812; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8813; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8814; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8815; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8816; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8817; GFX10-WGP-NEXT:    s_endpgm
8818;
8819; GFX10-CU-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
8820; GFX10-CU:       ; %bb.0: ; %entry
8821; GFX10-CU-NEXT:    s_clause 0x1
8822; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8823; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8824; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8825; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8826; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8827; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8828; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8829; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8830; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8831; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8832; GFX10-CU-NEXT:    s_endpgm
8833;
8834; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
8835; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8836; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8837; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8838; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8839; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8840; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8841; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8842; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8843; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8844; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8845; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8846; SKIP-CACHE-INV-NEXT:    s_endpgm
8847;
8848; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
8849; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8850; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8851; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8852; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8853; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8854; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8855; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8856; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8857;
8858; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
8859; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8860; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8861; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8862; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8863; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8864; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8865; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8866; GFX90A-TGSPLIT-NEXT:    s_endpgm
8867;
8868; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
8869; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8870; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8871; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8872; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8873; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8874; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8875; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8876; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8877;
8878; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
8879; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8880; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8881; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8882; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8883; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8884; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8885; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8886; GFX940-TGSPLIT-NEXT:    s_endpgm
8887;
8888; GFX11-WGP-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
8889; GFX11-WGP:       ; %bb.0: ; %entry
8890; GFX11-WGP-NEXT:    s_clause 0x1
8891; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8892; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8893; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8894; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8895; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
8896; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8897; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8898; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8899; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8900; GFX11-WGP-NEXT:    s_endpgm
8901;
8902; GFX11-CU-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
8903; GFX11-CU:       ; %bb.0: ; %entry
8904; GFX11-CU-NEXT:    s_clause 0x1
8905; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8906; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8907; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8908; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8909; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
8910; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8911; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8912; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8913; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8914; GFX11-CU-NEXT:    s_endpgm
8915    i32* %out, i32 %in, i32 %old) {
8916entry:
8917  %gep = getelementptr i32, i32* %out, i32 4
8918  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
8919  ret void
8920}
8921
8922define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_cmpxchg(
8923; GFX7-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8924; GFX7:       ; %bb.0: ; %entry
8925; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8926; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8927; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8928; GFX7-NEXT:    s_add_u32 s0, s0, 16
8929; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8930; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8931; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8932; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8933; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8934; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8935; GFX7-NEXT:    s_endpgm
8936;
8937; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8938; GFX10-WGP:       ; %bb.0: ; %entry
8939; GFX10-WGP-NEXT:    s_clause 0x1
8940; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8941; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8942; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8943; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8944; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8945; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8946; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8947; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8948; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8949; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8950; GFX10-WGP-NEXT:    s_endpgm
8951;
8952; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8953; GFX10-CU:       ; %bb.0: ; %entry
8954; GFX10-CU-NEXT:    s_clause 0x1
8955; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8956; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8957; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8958; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8959; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8960; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8961; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8962; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8963; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8964; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8965; GFX10-CU-NEXT:    s_endpgm
8966;
8967; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8968; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8969; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8970; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8971; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8972; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8973; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8974; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8975; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8976; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8977; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8978; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8979; SKIP-CACHE-INV-NEXT:    s_endpgm
8980;
8981; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8982; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8983; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8984; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8985; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8986; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8987; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8988; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8989; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8990;
8991; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8992; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8993; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8994; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8995; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8996; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8997; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8998; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8999; GFX90A-TGSPLIT-NEXT:    s_endpgm
9000;
9001; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
9002; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9003; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9004; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9005; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9006; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9007; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9008; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9009; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9010;
9011; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
9012; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9013; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9014; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9015; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9016; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9017; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9018; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9019; GFX940-TGSPLIT-NEXT:    s_endpgm
9020;
9021; GFX11-WGP-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
9022; GFX11-WGP:       ; %bb.0: ; %entry
9023; GFX11-WGP-NEXT:    s_clause 0x1
9024; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9025; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9026; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9027; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9028; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9029; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9030; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9031; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9032; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9033; GFX11-WGP-NEXT:    s_endpgm
9034;
9035; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
9036; GFX11-CU:       ; %bb.0: ; %entry
9037; GFX11-CU-NEXT:    s_clause 0x1
9038; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9039; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9040; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9041; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9042; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9043; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9044; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9045; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9046; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9047; GFX11-CU-NEXT:    s_endpgm
9048    i32* %out, i32 %in, i32 %old) {
9049entry:
9050  %gep = getelementptr i32, i32* %out, i32 4
9051  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
9052  ret void
9053}
9054
9055define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_cmpxchg(
9056; GFX7-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9057; GFX7:       ; %bb.0: ; %entry
9058; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9059; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9060; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9061; GFX7-NEXT:    s_add_u32 s0, s0, 16
9062; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9063; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9064; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9065; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9066; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9067; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9068; GFX7-NEXT:    s_endpgm
9069;
9070; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9071; GFX10-WGP:       ; %bb.0: ; %entry
9072; GFX10-WGP-NEXT:    s_clause 0x1
9073; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9074; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9075; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9076; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9077; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9078; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9079; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9080; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9081; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9082; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9083; GFX10-WGP-NEXT:    s_endpgm
9084;
9085; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9086; GFX10-CU:       ; %bb.0: ; %entry
9087; GFX10-CU-NEXT:    s_clause 0x1
9088; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9089; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9090; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9091; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9092; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9093; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9094; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9095; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9096; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9097; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9098; GFX10-CU-NEXT:    s_endpgm
9099;
9100; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9101; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9102; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9103; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9104; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9105; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9106; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9107; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9108; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9109; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9110; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9111; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9112; SKIP-CACHE-INV-NEXT:    s_endpgm
9113;
9114; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9115; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9116; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9117; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9118; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9119; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9120; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9121; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9122; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9123;
9124; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9125; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9126; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9127; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9128; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9129; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9130; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9131; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9132; GFX90A-TGSPLIT-NEXT:    s_endpgm
9133;
9134; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9135; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9136; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9137; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9138; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9139; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9140; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9141; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9142; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9143;
9144; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9145; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9146; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9147; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9148; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9149; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9150; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9151; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9152; GFX940-TGSPLIT-NEXT:    s_endpgm
9153;
9154; GFX11-WGP-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9155; GFX11-WGP:       ; %bb.0: ; %entry
9156; GFX11-WGP-NEXT:    s_clause 0x1
9157; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9158; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9159; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9160; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9161; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9162; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9163; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9164; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9165; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9166; GFX11-WGP-NEXT:    s_endpgm
9167;
9168; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9169; GFX11-CU:       ; %bb.0: ; %entry
9170; GFX11-CU-NEXT:    s_clause 0x1
9171; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9172; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9173; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9174; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9175; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9176; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9177; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9178; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9179; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9180; GFX11-CU-NEXT:    s_endpgm
9181    i32* %out, i32 %in, i32 %old) {
9182entry:
9183  %gep = getelementptr i32, i32* %out, i32 4
9184  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
9185  ret void
9186}
9187
9188define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_cmpxchg(
9189; GFX7-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
9190; GFX7:       ; %bb.0: ; %entry
9191; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9192; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9193; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9194; GFX7-NEXT:    s_add_u32 s0, s0, 16
9195; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9196; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9197; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9198; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9199; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9200; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9201; GFX7-NEXT:    s_endpgm
9202;
9203; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
9204; GFX10-WGP:       ; %bb.0: ; %entry
9205; GFX10-WGP-NEXT:    s_clause 0x1
9206; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9207; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9208; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9209; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9210; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9211; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9212; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9213; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9214; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9215; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9216; GFX10-WGP-NEXT:    s_endpgm
9217;
9218; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
9219; GFX10-CU:       ; %bb.0: ; %entry
9220; GFX10-CU-NEXT:    s_clause 0x1
9221; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9222; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9223; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9224; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9225; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9226; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9227; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9228; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9229; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9230; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9231; GFX10-CU-NEXT:    s_endpgm
9232;
9233; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
9234; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9235; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9236; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9237; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9238; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9239; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9240; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9241; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9242; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9243; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9244; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9245; SKIP-CACHE-INV-NEXT:    s_endpgm
9246;
9247; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
9248; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9249; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9250; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9251; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9252; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9253; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9254; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9255; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9256;
9257; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
9258; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9259; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9260; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9261; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9262; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9263; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9264; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9265; GFX90A-TGSPLIT-NEXT:    s_endpgm
9266;
9267; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
9268; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9269; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9270; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9271; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9272; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9273; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9274; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9275; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9276;
9277; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
9278; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9279; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9280; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9281; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9282; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9283; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9284; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9285; GFX940-TGSPLIT-NEXT:    s_endpgm
9286;
9287; GFX11-WGP-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
9288; GFX11-WGP:       ; %bb.0: ; %entry
9289; GFX11-WGP-NEXT:    s_clause 0x1
9290; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9291; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9292; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9293; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9294; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9295; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9296; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9297; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9298; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9299; GFX11-WGP-NEXT:    s_endpgm
9300;
9301; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
9302; GFX11-CU:       ; %bb.0: ; %entry
9303; GFX11-CU-NEXT:    s_clause 0x1
9304; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9305; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9306; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9307; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9308; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9309; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9310; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9311; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9312; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9313; GFX11-CU-NEXT:    s_endpgm
9314    i32* %out, i32 %in, i32 %old) {
9315entry:
9316  %gep = getelementptr i32, i32* %out, i32 4
9317  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic acquire
9318  ret void
9319}
9320
9321define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_cmpxchg(
9322; GFX7-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
9323; GFX7:       ; %bb.0: ; %entry
9324; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9325; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9326; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9327; GFX7-NEXT:    s_add_u32 s0, s0, 16
9328; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9329; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9330; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9331; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9332; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9333; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9334; GFX7-NEXT:    s_endpgm
9335;
9336; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
9337; GFX10-WGP:       ; %bb.0: ; %entry
9338; GFX10-WGP-NEXT:    s_clause 0x1
9339; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9340; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9341; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9342; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9343; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9344; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9345; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9346; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9347; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9348; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9349; GFX10-WGP-NEXT:    s_endpgm
9350;
9351; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
9352; GFX10-CU:       ; %bb.0: ; %entry
9353; GFX10-CU-NEXT:    s_clause 0x1
9354; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9355; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9356; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9357; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9358; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9359; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9360; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9361; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9362; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9363; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9364; GFX10-CU-NEXT:    s_endpgm
9365;
9366; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
9367; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9368; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9369; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9370; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9371; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9372; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9373; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9374; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9375; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9376; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9377; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9378; SKIP-CACHE-INV-NEXT:    s_endpgm
9379;
9380; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
9381; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9382; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9383; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9384; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9385; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9386; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9387; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9388; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9389;
9390; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
9391; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9392; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9393; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9394; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9395; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9396; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9397; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9398; GFX90A-TGSPLIT-NEXT:    s_endpgm
9399;
9400; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
9401; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9402; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9403; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9404; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9405; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9406; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9407; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9408; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9409;
9410; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
9411; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9412; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9413; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9414; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9415; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9416; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9417; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9418; GFX940-TGSPLIT-NEXT:    s_endpgm
9419;
9420; GFX11-WGP-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
9421; GFX11-WGP:       ; %bb.0: ; %entry
9422; GFX11-WGP-NEXT:    s_clause 0x1
9423; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9424; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9425; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9426; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9427; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9428; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9429; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9430; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9431; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9432; GFX11-WGP-NEXT:    s_endpgm
9433;
9434; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
9435; GFX11-CU:       ; %bb.0: ; %entry
9436; GFX11-CU-NEXT:    s_clause 0x1
9437; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9438; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9439; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9440; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9441; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9442; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9443; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9444; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9445; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9446; GFX11-CU-NEXT:    s_endpgm
9447    i32* %out, i32 %in, i32 %old) {
9448entry:
9449  %gep = getelementptr i32, i32* %out, i32 4
9450  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
9451  ret void
9452}
9453
9454define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_cmpxchg(
9455; GFX7-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
9456; GFX7:       ; %bb.0: ; %entry
9457; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9458; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9459; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9460; GFX7-NEXT:    s_add_u32 s0, s0, 16
9461; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9462; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9463; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9464; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9465; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9466; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9467; GFX7-NEXT:    s_endpgm
9468;
9469; GFX10-WGP-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
9470; GFX10-WGP:       ; %bb.0: ; %entry
9471; GFX10-WGP-NEXT:    s_clause 0x1
9472; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9473; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9474; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9475; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9476; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9477; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9478; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9479; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9480; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9481; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9482; GFX10-WGP-NEXT:    s_endpgm
9483;
9484; GFX10-CU-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
9485; GFX10-CU:       ; %bb.0: ; %entry
9486; GFX10-CU-NEXT:    s_clause 0x1
9487; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9488; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9489; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9490; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9491; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9492; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9493; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9494; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9495; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9496; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9497; GFX10-CU-NEXT:    s_endpgm
9498;
9499; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
9500; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9501; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9502; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9503; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9504; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9505; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9506; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9507; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9508; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9509; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9510; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9511; SKIP-CACHE-INV-NEXT:    s_endpgm
9512;
9513; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
9514; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9515; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9516; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9517; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9518; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9519; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9520; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9521; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9522;
9523; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
9524; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9525; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9526; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9527; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9528; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9529; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9530; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9531; GFX90A-TGSPLIT-NEXT:    s_endpgm
9532;
9533; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
9534; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9535; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9536; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9537; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9538; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9539; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9540; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9541; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9542;
9543; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
9544; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9545; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9546; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9547; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9548; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9549; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9550; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9551; GFX940-TGSPLIT-NEXT:    s_endpgm
9552;
9553; GFX11-WGP-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
9554; GFX11-WGP:       ; %bb.0: ; %entry
9555; GFX11-WGP-NEXT:    s_clause 0x1
9556; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9557; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9558; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9559; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9560; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9561; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9562; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9563; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9564; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9565; GFX11-WGP-NEXT:    s_endpgm
9566;
9567; GFX11-CU-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
9568; GFX11-CU:       ; %bb.0: ; %entry
9569; GFX11-CU-NEXT:    s_clause 0x1
9570; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9571; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9572; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9573; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9574; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9575; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9576; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9577; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9578; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9579; GFX11-CU-NEXT:    s_endpgm
9580    i32* %out, i32 %in, i32 %old) {
9581entry:
9582  %gep = getelementptr i32, i32* %out, i32 4
9583  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
9584  ret void
9585}
9586
9587define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_cmpxchg(
9588; GFX7-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
9589; GFX7:       ; %bb.0: ; %entry
9590; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9591; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9592; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9593; GFX7-NEXT:    s_add_u32 s0, s0, 16
9594; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9595; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9596; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9597; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9598; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9599; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9600; GFX7-NEXT:    s_endpgm
9601;
9602; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
9603; GFX10-WGP:       ; %bb.0: ; %entry
9604; GFX10-WGP-NEXT:    s_clause 0x1
9605; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9606; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9607; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9608; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9609; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9610; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9611; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9612; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9613; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9614; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9615; GFX10-WGP-NEXT:    s_endpgm
9616;
9617; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
9618; GFX10-CU:       ; %bb.0: ; %entry
9619; GFX10-CU-NEXT:    s_clause 0x1
9620; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9621; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9622; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9623; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9624; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9625; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9626; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9627; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9628; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9629; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9630; GFX10-CU-NEXT:    s_endpgm
9631;
9632; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
9633; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9634; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9635; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9636; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9637; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9638; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9639; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9640; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9641; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9642; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9643; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9644; SKIP-CACHE-INV-NEXT:    s_endpgm
9645;
9646; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
9647; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9648; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9649; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9650; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9651; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9652; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9653; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9654; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9655;
9656; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
9657; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9658; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9659; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9660; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9661; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9662; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9663; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9664; GFX90A-TGSPLIT-NEXT:    s_endpgm
9665;
9666; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
9667; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9668; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9669; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9670; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9671; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9672; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9673; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9674; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9675;
9676; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
9677; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9678; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9679; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9680; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9681; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9682; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9683; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9684; GFX940-TGSPLIT-NEXT:    s_endpgm
9685;
9686; GFX11-WGP-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
9687; GFX11-WGP:       ; %bb.0: ; %entry
9688; GFX11-WGP-NEXT:    s_clause 0x1
9689; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9690; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9691; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9692; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9693; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9694; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9695; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9696; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9697; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9698; GFX11-WGP-NEXT:    s_endpgm
9699;
9700; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
9701; GFX11-CU:       ; %bb.0: ; %entry
9702; GFX11-CU-NEXT:    s_clause 0x1
9703; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9704; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9705; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9706; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9707; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9708; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9709; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9710; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9711; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9712; GFX11-CU-NEXT:    s_endpgm
9713    i32* %out, i32 %in, i32 %old) {
9714entry:
9715  %gep = getelementptr i32, i32* %out, i32 4
9716  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
9717  ret void
9718}
9719
9720define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_cmpxchg(
9721; GFX7-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
9722; GFX7:       ; %bb.0: ; %entry
9723; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9724; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9725; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9726; GFX7-NEXT:    s_add_u32 s0, s0, 16
9727; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9728; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9729; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9730; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9731; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9732; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9733; GFX7-NEXT:    s_endpgm
9734;
9735; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
9736; GFX10-WGP:       ; %bb.0: ; %entry
9737; GFX10-WGP-NEXT:    s_clause 0x1
9738; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9739; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9740; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9741; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9742; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9743; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9744; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9745; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9746; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9747; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9748; GFX10-WGP-NEXT:    s_endpgm
9749;
9750; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
9751; GFX10-CU:       ; %bb.0: ; %entry
9752; GFX10-CU-NEXT:    s_clause 0x1
9753; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9754; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9755; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9756; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9757; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9758; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9759; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9760; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9761; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9762; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9763; GFX10-CU-NEXT:    s_endpgm
9764;
9765; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
9766; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9767; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9768; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9769; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9770; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9771; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9772; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9773; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9774; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9775; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9776; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9777; SKIP-CACHE-INV-NEXT:    s_endpgm
9778;
9779; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
9780; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9781; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9782; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9783; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9784; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9785; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9786; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9787; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9788;
9789; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
9790; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9791; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9792; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9793; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9794; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9795; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9796; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9797; GFX90A-TGSPLIT-NEXT:    s_endpgm
9798;
9799; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
9800; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9801; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9802; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9803; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9804; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9805; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9806; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9807; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9808;
9809; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
9810; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9811; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9812; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9813; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9814; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9815; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9816; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9817; GFX940-TGSPLIT-NEXT:    s_endpgm
9818;
9819; GFX11-WGP-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
9820; GFX11-WGP:       ; %bb.0: ; %entry
9821; GFX11-WGP-NEXT:    s_clause 0x1
9822; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9823; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9824; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9825; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9826; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9827; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9828; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9829; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9830; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9831; GFX11-WGP-NEXT:    s_endpgm
9832;
9833; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
9834; GFX11-CU:       ; %bb.0: ; %entry
9835; GFX11-CU-NEXT:    s_clause 0x1
9836; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9837; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9838; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9839; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9840; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9841; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9842; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9843; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9844; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9845; GFX11-CU-NEXT:    s_endpgm
9846    i32* %out, i32 %in, i32 %old) {
9847entry:
9848  %gep = getelementptr i32, i32* %out, i32 4
9849  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
9850  ret void
9851}
9852
9853define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_cmpxchg(
9854; GFX7-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9855; GFX7:       ; %bb.0: ; %entry
9856; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9857; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9858; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9859; GFX7-NEXT:    s_add_u32 s0, s0, 16
9860; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9861; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9862; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9863; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9864; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9865; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9866; GFX7-NEXT:    s_endpgm
9867;
9868; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9869; GFX10-WGP:       ; %bb.0: ; %entry
9870; GFX10-WGP-NEXT:    s_clause 0x1
9871; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9872; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9873; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9874; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9875; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9876; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9877; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9878; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9879; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9880; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9881; GFX10-WGP-NEXT:    s_endpgm
9882;
9883; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9884; GFX10-CU:       ; %bb.0: ; %entry
9885; GFX10-CU-NEXT:    s_clause 0x1
9886; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9887; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9888; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9889; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9890; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9891; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9892; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9893; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9894; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9895; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9896; GFX10-CU-NEXT:    s_endpgm
9897;
9898; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9899; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9900; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9901; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9902; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9903; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9904; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9905; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9906; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9907; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9908; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9909; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9910; SKIP-CACHE-INV-NEXT:    s_endpgm
9911;
9912; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9913; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9914; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9915; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9916; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9917; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9918; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9919; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9920; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9921;
9922; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9923; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9924; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9925; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9926; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9927; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9928; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9929; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9930; GFX90A-TGSPLIT-NEXT:    s_endpgm
9931;
9932; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9933; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9934; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9935; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9936; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9937; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9938; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9939; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9940; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9941;
9942; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9943; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9944; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9945; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9946; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9947; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9948; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9949; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9950; GFX940-TGSPLIT-NEXT:    s_endpgm
9951;
9952; GFX11-WGP-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9953; GFX11-WGP:       ; %bb.0: ; %entry
9954; GFX11-WGP-NEXT:    s_clause 0x1
9955; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9956; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9957; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9958; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9959; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9960; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9961; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9962; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9963; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9964; GFX11-WGP-NEXT:    s_endpgm
9965;
9966; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9967; GFX11-CU:       ; %bb.0: ; %entry
9968; GFX11-CU-NEXT:    s_clause 0x1
9969; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9970; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9971; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9972; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9973; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9974; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9975; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9976; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9977; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9978; GFX11-CU-NEXT:    s_endpgm
9979    i32* %out, i32 %in, i32 %old) {
9980entry:
9981  %gep = getelementptr i32, i32* %out, i32 4
9982  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic seq_cst
9983  ret void
9984}
9985
9986define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_cmpxchg(
9987; GFX7-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
9988; GFX7:       ; %bb.0: ; %entry
9989; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9990; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9991; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9992; GFX7-NEXT:    s_add_u32 s0, s0, 16
9993; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9994; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9995; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9996; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9997; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9998; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9999; GFX7-NEXT:    s_endpgm
10000;
10001; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
10002; GFX10-WGP:       ; %bb.0: ; %entry
10003; GFX10-WGP-NEXT:    s_clause 0x1
10004; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10005; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10006; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10007; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10008; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10009; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10010; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10011; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10012; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10013; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10014; GFX10-WGP-NEXT:    s_endpgm
10015;
10016; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
10017; GFX10-CU:       ; %bb.0: ; %entry
10018; GFX10-CU-NEXT:    s_clause 0x1
10019; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10020; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10021; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10022; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10023; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10024; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10025; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10026; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10027; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10028; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10029; GFX10-CU-NEXT:    s_endpgm
10030;
10031; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
10032; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10033; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10034; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10035; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10036; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10037; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10038; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10039; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10040; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10041; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10042; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10043; SKIP-CACHE-INV-NEXT:    s_endpgm
10044;
10045; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
10046; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10047; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10048; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10049; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10050; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10051; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10052; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10053; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10054;
10055; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
10056; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10057; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10058; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10059; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10060; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10061; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10062; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10063; GFX90A-TGSPLIT-NEXT:    s_endpgm
10064;
10065; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
10066; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10067; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10068; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10069; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10070; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10071; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10072; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10073; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10074;
10075; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
10076; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10077; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10078; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10079; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10080; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10081; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10082; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10083; GFX940-TGSPLIT-NEXT:    s_endpgm
10084;
10085; GFX11-WGP-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
10086; GFX11-WGP:       ; %bb.0: ; %entry
10087; GFX11-WGP-NEXT:    s_clause 0x1
10088; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10089; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10090; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10091; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10092; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10093; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10094; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10095; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10096; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10097; GFX11-WGP-NEXT:    s_endpgm
10098;
10099; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
10100; GFX11-CU:       ; %bb.0: ; %entry
10101; GFX11-CU-NEXT:    s_clause 0x1
10102; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10103; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10104; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10105; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10106; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10107; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10108; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10109; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10110; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10111; GFX11-CU-NEXT:    s_endpgm
10112    i32* %out, i32 %in, i32 %old) {
10113entry:
10114  %gep = getelementptr i32, i32* %out, i32 4
10115  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire seq_cst
10116  ret void
10117}
10118
10119define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_cmpxchg(
10120; GFX7-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
10121; GFX7:       ; %bb.0: ; %entry
10122; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10123; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10124; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10125; GFX7-NEXT:    s_add_u32 s0, s0, 16
10126; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10127; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10128; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10129; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10130; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10131; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10132; GFX7-NEXT:    s_endpgm
10133;
10134; GFX10-WGP-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
10135; GFX10-WGP:       ; %bb.0: ; %entry
10136; GFX10-WGP-NEXT:    s_clause 0x1
10137; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10138; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10139; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10140; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10141; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10142; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10143; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10144; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10145; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10146; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10147; GFX10-WGP-NEXT:    s_endpgm
10148;
10149; GFX10-CU-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
10150; GFX10-CU:       ; %bb.0: ; %entry
10151; GFX10-CU-NEXT:    s_clause 0x1
10152; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10153; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10154; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10155; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10156; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10157; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10158; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10159; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10160; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10161; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10162; GFX10-CU-NEXT:    s_endpgm
10163;
10164; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
10165; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10166; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10167; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10168; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10169; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10170; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10171; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10172; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10173; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10174; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10175; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10176; SKIP-CACHE-INV-NEXT:    s_endpgm
10177;
10178; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
10179; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10180; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10181; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10182; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10183; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10184; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10185; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10186; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10187;
10188; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
10189; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10190; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10191; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10192; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10193; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10194; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10195; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10196; GFX90A-TGSPLIT-NEXT:    s_endpgm
10197;
10198; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
10199; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10200; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10201; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10202; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10203; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10204; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10205; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10206; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10207;
10208; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
10209; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10210; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10211; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10212; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10213; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10214; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10215; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10216; GFX940-TGSPLIT-NEXT:    s_endpgm
10217;
10218; GFX11-WGP-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
10219; GFX11-WGP:       ; %bb.0: ; %entry
10220; GFX11-WGP-NEXT:    s_clause 0x1
10221; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10222; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10223; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10224; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10225; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10226; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10227; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10228; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10229; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10230; GFX11-WGP-NEXT:    s_endpgm
10231;
10232; GFX11-CU-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
10233; GFX11-CU:       ; %bb.0: ; %entry
10234; GFX11-CU-NEXT:    s_clause 0x1
10235; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10236; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10237; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10238; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10239; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10240; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10241; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10242; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10243; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10244; GFX11-CU-NEXT:    s_endpgm
10245    i32* %out, i32 %in, i32 %old) {
10246entry:
10247  %gep = getelementptr i32, i32* %out, i32 4
10248  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release seq_cst
10249  ret void
10250}
10251
10252define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
10253; GFX7-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10254; GFX7:       ; %bb.0: ; %entry
10255; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10256; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10257; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10258; GFX7-NEXT:    s_add_u32 s0, s0, 16
10259; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10260; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10261; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10262; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10263; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10264; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10265; GFX7-NEXT:    s_endpgm
10266;
10267; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10268; GFX10-WGP:       ; %bb.0: ; %entry
10269; GFX10-WGP-NEXT:    s_clause 0x1
10270; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10271; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10272; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10273; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10274; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10275; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10276; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10277; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10278; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10279; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10280; GFX10-WGP-NEXT:    s_endpgm
10281;
10282; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10283; GFX10-CU:       ; %bb.0: ; %entry
10284; GFX10-CU-NEXT:    s_clause 0x1
10285; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10286; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10287; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10288; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10289; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10290; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10291; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10292; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10293; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10294; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10295; GFX10-CU-NEXT:    s_endpgm
10296;
10297; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10298; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10299; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10300; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10301; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10302; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10303; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10304; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10305; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10306; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10307; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10308; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10309; SKIP-CACHE-INV-NEXT:    s_endpgm
10310;
10311; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10312; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10313; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10314; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10315; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10316; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10317; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10318; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10319; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10320;
10321; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10322; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10323; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10324; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10325; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10326; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10327; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10328; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10329; GFX90A-TGSPLIT-NEXT:    s_endpgm
10330;
10331; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10332; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10333; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10334; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10335; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10336; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10337; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10338; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10339; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10340;
10341; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10342; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10343; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10344; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10345; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10346; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10347; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10348; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10349; GFX940-TGSPLIT-NEXT:    s_endpgm
10350;
10351; GFX11-WGP-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10352; GFX11-WGP:       ; %bb.0: ; %entry
10353; GFX11-WGP-NEXT:    s_clause 0x1
10354; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10355; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10356; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10357; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10358; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10359; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10360; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10361; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10362; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10363; GFX11-WGP-NEXT:    s_endpgm
10364;
10365; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10366; GFX11-CU:       ; %bb.0: ; %entry
10367; GFX11-CU-NEXT:    s_clause 0x1
10368; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10369; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10370; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10371; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10372; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10373; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10374; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10375; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10376; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10377; GFX11-CU-NEXT:    s_endpgm
10378    i32* %out, i32 %in, i32 %old) {
10379entry:
10380  %gep = getelementptr i32, i32* %out, i32 4
10381  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel seq_cst
10382  ret void
10383}
10384
10385define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
10386; GFX7-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10387; GFX7:       ; %bb.0: ; %entry
10388; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10389; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10390; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10391; GFX7-NEXT:    s_add_u32 s0, s0, 16
10392; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10393; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10394; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10395; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10396; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10397; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10398; GFX7-NEXT:    s_endpgm
10399;
10400; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10401; GFX10-WGP:       ; %bb.0: ; %entry
10402; GFX10-WGP-NEXT:    s_clause 0x1
10403; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10404; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10405; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10406; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10407; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10408; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10409; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10410; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10411; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10412; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10413; GFX10-WGP-NEXT:    s_endpgm
10414;
10415; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10416; GFX10-CU:       ; %bb.0: ; %entry
10417; GFX10-CU-NEXT:    s_clause 0x1
10418; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10419; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10420; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10421; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10422; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10423; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10424; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10425; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10426; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10427; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10428; GFX10-CU-NEXT:    s_endpgm
10429;
10430; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10431; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10432; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10433; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10434; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10435; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10436; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10437; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10438; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10439; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10440; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10441; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10442; SKIP-CACHE-INV-NEXT:    s_endpgm
10443;
10444; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10445; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10446; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10447; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10448; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10449; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10450; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10451; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10452; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10453;
10454; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10455; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10456; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10457; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10458; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10459; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10460; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10461; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10462; GFX90A-TGSPLIT-NEXT:    s_endpgm
10463;
10464; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10465; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10466; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10467; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10468; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10469; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10470; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10471; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10472; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10473;
10474; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10475; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10476; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10477; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10478; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10479; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10480; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10481; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10482; GFX940-TGSPLIT-NEXT:    s_endpgm
10483;
10484; GFX11-WGP-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10485; GFX11-WGP:       ; %bb.0: ; %entry
10486; GFX11-WGP-NEXT:    s_clause 0x1
10487; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10488; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10489; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10490; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10491; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10492; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10493; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10494; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10495; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10496; GFX11-WGP-NEXT:    s_endpgm
10497;
10498; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10499; GFX11-CU:       ; %bb.0: ; %entry
10500; GFX11-CU-NEXT:    s_clause 0x1
10501; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10502; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10503; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10504; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10505; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10506; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10507; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10508; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10509; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10510; GFX11-CU-NEXT:    s_endpgm
10511    i32* %out, i32 %in, i32 %old) {
10512entry:
10513  %gep = getelementptr i32, i32* %out, i32 4
10514  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
10515  ret void
10516}
10517
10518define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg(
10519; GFX7-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10520; GFX7:       ; %bb.0: ; %entry
10521; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10522; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10523; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10524; GFX7-NEXT:    s_add_u32 s4, s0, 16
10525; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10526; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10527; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10528; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10529; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10530; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10531; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10532; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10533; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10534; GFX7-NEXT:    flat_store_dword v[0:1], v2
10535; GFX7-NEXT:    s_endpgm
10536;
10537; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10538; GFX10-WGP:       ; %bb.0: ; %entry
10539; GFX10-WGP-NEXT:    s_clause 0x1
10540; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10541; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10542; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10543; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10544; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10545; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10546; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10547; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10548; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10549; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10550; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10551; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10552; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10553; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10554; GFX10-WGP-NEXT:    s_endpgm
10555;
10556; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10557; GFX10-CU:       ; %bb.0: ; %entry
10558; GFX10-CU-NEXT:    s_clause 0x1
10559; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10560; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10561; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10562; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10563; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10564; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10565; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10566; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10567; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10568; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10569; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10570; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10571; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10572; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10573; GFX10-CU-NEXT:    s_endpgm
10574;
10575; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10576; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10577; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10578; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10579; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10580; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10581; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10582; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10583; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10584; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10585; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10586; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10587; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10588; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10589; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10590; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10591; SKIP-CACHE-INV-NEXT:    s_endpgm
10592;
10593; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10594; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10595; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10596; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10597; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10598; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10599; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10600; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10601; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10602; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10603; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10604;
10605; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10606; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10607; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10608; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10609; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10610; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10611; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10612; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10613; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10614; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10615; GFX90A-TGSPLIT-NEXT:    s_endpgm
10616;
10617; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10618; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10619; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10620; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10621; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10622; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10623; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10624; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10625; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10626; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10627; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10628;
10629; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10630; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10631; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10632; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10633; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10634; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10635; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10636; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10637; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10638; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10639; GFX940-TGSPLIT-NEXT:    s_endpgm
10640;
10641; GFX11-WGP-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10642; GFX11-WGP:       ; %bb.0: ; %entry
10643; GFX11-WGP-NEXT:    s_clause 0x1
10644; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10645; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10646; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10647; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10648; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10649; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10650; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10651; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10652; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10653; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
10654; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10655; GFX11-WGP-NEXT:    s_endpgm
10656;
10657; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10658; GFX11-CU:       ; %bb.0: ; %entry
10659; GFX11-CU-NEXT:    s_clause 0x1
10660; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10661; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10662; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10663; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10664; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10665; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10666; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10667; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10668; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10669; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
10670; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10671; GFX11-CU-NEXT:    s_endpgm
10672    i32* %out, i32 %in, i32 %old) {
10673entry:
10674  %gep = getelementptr i32, i32* %out, i32 4
10675  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
10676  %val0 = extractvalue { i32, i1 } %val, 0
10677  store i32 %val0, i32* %out, align 4
10678  ret void
10679}
10680
10681define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg(
10682; GFX7-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10683; GFX7:       ; %bb.0: ; %entry
10684; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10685; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10686; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10687; GFX7-NEXT:    s_add_u32 s4, s0, 16
10688; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10689; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10690; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10691; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10692; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10693; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10694; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10695; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10696; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10697; GFX7-NEXT:    flat_store_dword v[0:1], v2
10698; GFX7-NEXT:    s_endpgm
10699;
10700; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10701; GFX10-WGP:       ; %bb.0: ; %entry
10702; GFX10-WGP-NEXT:    s_clause 0x1
10703; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10704; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10705; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10706; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10707; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10708; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10709; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10710; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10711; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10712; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10713; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10714; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10715; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10716; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10717; GFX10-WGP-NEXT:    s_endpgm
10718;
10719; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10720; GFX10-CU:       ; %bb.0: ; %entry
10721; GFX10-CU-NEXT:    s_clause 0x1
10722; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10723; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10724; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10725; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10726; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10727; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10728; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10729; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10730; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10731; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10732; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10733; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10734; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10735; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10736; GFX10-CU-NEXT:    s_endpgm
10737;
10738; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10739; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10740; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10741; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10742; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10743; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10744; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10745; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10746; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10747; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10748; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10749; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10750; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10751; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10752; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10753; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10754; SKIP-CACHE-INV-NEXT:    s_endpgm
10755;
10756; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10757; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10758; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10759; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10760; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10761; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10762; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10763; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10764; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10765; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10766; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10767;
10768; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10769; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10770; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10771; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10772; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10773; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10774; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10775; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10776; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10777; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10778; GFX90A-TGSPLIT-NEXT:    s_endpgm
10779;
10780; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10781; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10782; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10783; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10784; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10785; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10786; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10787; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10788; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10789; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10790; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10791;
10792; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10793; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10794; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10795; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10796; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10797; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10798; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10799; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10800; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10801; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10802; GFX940-TGSPLIT-NEXT:    s_endpgm
10803;
10804; GFX11-WGP-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10805; GFX11-WGP:       ; %bb.0: ; %entry
10806; GFX11-WGP-NEXT:    s_clause 0x1
10807; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10808; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10809; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10810; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10811; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10812; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10813; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10814; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10815; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10816; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
10817; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10818; GFX11-WGP-NEXT:    s_endpgm
10819;
10820; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10821; GFX11-CU:       ; %bb.0: ; %entry
10822; GFX11-CU-NEXT:    s_clause 0x1
10823; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10824; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10825; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10826; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10827; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10828; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10829; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10830; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10831; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10832; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
10833; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10834; GFX11-CU-NEXT:    s_endpgm
10835    i32* %out, i32 %in, i32 %old) {
10836entry:
10837  %gep = getelementptr i32, i32* %out, i32 4
10838  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
10839  %val0 = extractvalue { i32, i1 } %val, 0
10840  store i32 %val0, i32* %out, align 4
10841  ret void
10842}
10843
10844define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_ret_cmpxchg(
10845; GFX7-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
10846; GFX7:       ; %bb.0: ; %entry
10847; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10848; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10849; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10850; GFX7-NEXT:    s_add_u32 s4, s0, 16
10851; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10852; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10853; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10854; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10855; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10856; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10857; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10858; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10859; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10860; GFX7-NEXT:    flat_store_dword v[0:1], v2
10861; GFX7-NEXT:    s_endpgm
10862;
10863; GFX10-WGP-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
10864; GFX10-WGP:       ; %bb.0: ; %entry
10865; GFX10-WGP-NEXT:    s_clause 0x1
10866; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10867; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10868; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10869; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10870; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10871; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10872; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10873; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10874; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10875; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10876; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10877; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10878; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10879; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10880; GFX10-WGP-NEXT:    s_endpgm
10881;
10882; GFX10-CU-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
10883; GFX10-CU:       ; %bb.0: ; %entry
10884; GFX10-CU-NEXT:    s_clause 0x1
10885; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10886; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10887; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10888; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10889; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10890; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10891; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10892; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10893; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10894; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10895; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10896; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10897; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10898; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10899; GFX10-CU-NEXT:    s_endpgm
10900;
10901; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
10902; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10903; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10904; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10905; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10906; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10907; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10908; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10909; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10910; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10911; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10912; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10913; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10914; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10915; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10916; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10917; SKIP-CACHE-INV-NEXT:    s_endpgm
10918;
10919; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
10920; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10921; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10922; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10923; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10924; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10925; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10926; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10927; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10928; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10929; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10930;
10931; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
10932; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10933; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10934; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10935; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10936; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10937; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10938; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10939; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10940; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10941; GFX90A-TGSPLIT-NEXT:    s_endpgm
10942;
10943; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
10944; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10945; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10946; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10947; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10948; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10949; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10950; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10951; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10952; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10953; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10954;
10955; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
10956; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10957; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10958; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10959; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10960; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10961; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10962; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10963; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10964; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10965; GFX940-TGSPLIT-NEXT:    s_endpgm
10966;
10967; GFX11-WGP-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
10968; GFX11-WGP:       ; %bb.0: ; %entry
10969; GFX11-WGP-NEXT:    s_clause 0x1
10970; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10971; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10972; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10973; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10974; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10975; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10976; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10977; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10978; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10979; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
10980; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10981; GFX11-WGP-NEXT:    s_endpgm
10982;
10983; GFX11-CU-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
10984; GFX11-CU:       ; %bb.0: ; %entry
10985; GFX11-CU-NEXT:    s_clause 0x1
10986; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10987; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10988; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10989; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10990; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10991; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10992; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10993; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10994; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10995; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
10996; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10997; GFX11-CU-NEXT:    s_endpgm
10998    i32* %out, i32 %in, i32 %old) {
10999entry:
11000  %gep = getelementptr i32, i32* %out, i32 4
11001  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
11002  %val0 = extractvalue { i32, i1 } %val, 0
11003  store i32 %val0, i32* %out, align 4
11004  ret void
11005}
11006
11007define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg(
11008; GFX7-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11009; GFX7:       ; %bb.0: ; %entry
11010; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11011; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11012; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11013; GFX7-NEXT:    s_add_u32 s4, s0, 16
11014; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11015; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11016; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11017; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11018; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11019; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11020; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11021; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11022; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11023; GFX7-NEXT:    flat_store_dword v[0:1], v2
11024; GFX7-NEXT:    s_endpgm
11025;
11026; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11027; GFX10-WGP:       ; %bb.0: ; %entry
11028; GFX10-WGP-NEXT:    s_clause 0x1
11029; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11030; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11031; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11032; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11033; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11034; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11035; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11036; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11037; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11038; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11039; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11040; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11041; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11042; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11043; GFX10-WGP-NEXT:    s_endpgm
11044;
11045; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11046; GFX10-CU:       ; %bb.0: ; %entry
11047; GFX10-CU-NEXT:    s_clause 0x1
11048; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11049; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11050; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11051; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11052; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11053; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11054; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11055; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11056; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11057; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11058; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11059; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11060; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11061; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11062; GFX10-CU-NEXT:    s_endpgm
11063;
11064; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11065; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11066; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11067; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11068; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11069; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11070; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11071; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11072; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11073; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11074; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11075; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11076; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11077; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11078; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11079; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11080; SKIP-CACHE-INV-NEXT:    s_endpgm
11081;
11082; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11083; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11084; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11085; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11086; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11087; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11088; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11089; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11090; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11091; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11092; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11093;
11094; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11095; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11096; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11097; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11098; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11099; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11100; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11101; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11102; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11103; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11104; GFX90A-TGSPLIT-NEXT:    s_endpgm
11105;
11106; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11107; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11108; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11109; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11110; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11111; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11112; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11113; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11114; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11115; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11116; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11117;
11118; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11119; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11120; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11121; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11122; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11123; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11124; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11125; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11126; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11127; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11128; GFX940-TGSPLIT-NEXT:    s_endpgm
11129;
11130; GFX11-WGP-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11131; GFX11-WGP:       ; %bb.0: ; %entry
11132; GFX11-WGP-NEXT:    s_clause 0x1
11133; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11134; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11135; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11136; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11137; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11138; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11139; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11140; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11141; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11142; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11143; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11144; GFX11-WGP-NEXT:    s_endpgm
11145;
11146; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11147; GFX11-CU:       ; %bb.0: ; %entry
11148; GFX11-CU-NEXT:    s_clause 0x1
11149; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11150; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11151; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11152; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11153; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11154; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11155; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11156; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11157; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11158; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11159; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11160; GFX11-CU-NEXT:    s_endpgm
11161    i32* %out, i32 %in, i32 %old) {
11162entry:
11163  %gep = getelementptr i32, i32* %out, i32 4
11164  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
11165  %val0 = extractvalue { i32, i1 } %val, 0
11166  store i32 %val0, i32* %out, align 4
11167  ret void
11168}
11169
11170define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg(
11171; GFX7-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11172; GFX7:       ; %bb.0: ; %entry
11173; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11174; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11175; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11176; GFX7-NEXT:    s_add_u32 s4, s0, 16
11177; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11178; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11179; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11180; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11181; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11182; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11183; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11184; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11185; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11186; GFX7-NEXT:    flat_store_dword v[0:1], v2
11187; GFX7-NEXT:    s_endpgm
11188;
11189; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11190; GFX10-WGP:       ; %bb.0: ; %entry
11191; GFX10-WGP-NEXT:    s_clause 0x1
11192; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11193; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11194; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11195; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11196; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11197; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11198; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11199; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11200; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11201; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11202; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11203; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11204; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11205; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11206; GFX10-WGP-NEXT:    s_endpgm
11207;
11208; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11209; GFX10-CU:       ; %bb.0: ; %entry
11210; GFX10-CU-NEXT:    s_clause 0x1
11211; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11212; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11213; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11214; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11215; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11216; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11217; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11218; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11219; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11220; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11221; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11222; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11223; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11224; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11225; GFX10-CU-NEXT:    s_endpgm
11226;
11227; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11228; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11229; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11230; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11231; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11232; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11233; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11234; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11235; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11236; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11237; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11238; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11239; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11240; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11241; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11242; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11243; SKIP-CACHE-INV-NEXT:    s_endpgm
11244;
11245; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11246; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11247; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11248; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11249; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11250; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11251; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11252; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11253; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11254; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11255; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11256;
11257; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11258; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11259; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11260; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11261; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11262; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11263; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11264; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11265; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11266; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11267; GFX90A-TGSPLIT-NEXT:    s_endpgm
11268;
11269; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11270; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11271; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11272; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11273; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11274; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11275; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11276; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11277; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11278; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11279; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11280;
11281; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11282; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11283; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11284; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11285; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11286; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11287; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11288; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11289; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11290; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11291; GFX940-TGSPLIT-NEXT:    s_endpgm
11292;
11293; GFX11-WGP-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11294; GFX11-WGP:       ; %bb.0: ; %entry
11295; GFX11-WGP-NEXT:    s_clause 0x1
11296; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11297; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11298; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11299; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11300; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11301; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11302; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11303; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11304; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11305; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11306; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11307; GFX11-WGP-NEXT:    s_endpgm
11308;
11309; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11310; GFX11-CU:       ; %bb.0: ; %entry
11311; GFX11-CU-NEXT:    s_clause 0x1
11312; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11313; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11314; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11315; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11316; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11317; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11318; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11319; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11320; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11321; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11322; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11323; GFX11-CU-NEXT:    s_endpgm
11324    i32* %out, i32 %in, i32 %old) {
11325entry:
11326  %gep = getelementptr i32, i32* %out, i32 4
11327  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
11328  %val0 = extractvalue { i32, i1 } %val, 0
11329  store i32 %val0, i32* %out, align 4
11330  ret void
11331}
11332
11333define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg(
11334; GFX7-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11335; GFX7:       ; %bb.0: ; %entry
11336; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11337; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11338; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11339; GFX7-NEXT:    s_add_u32 s4, s0, 16
11340; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11341; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11342; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11343; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11344; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11345; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11346; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11347; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11348; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11349; GFX7-NEXT:    flat_store_dword v[0:1], v2
11350; GFX7-NEXT:    s_endpgm
11351;
11352; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11353; GFX10-WGP:       ; %bb.0: ; %entry
11354; GFX10-WGP-NEXT:    s_clause 0x1
11355; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11356; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11357; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11358; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11359; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11360; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11361; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11362; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11363; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11364; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11365; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11366; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11367; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11368; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11369; GFX10-WGP-NEXT:    s_endpgm
11370;
11371; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11372; GFX10-CU:       ; %bb.0: ; %entry
11373; GFX10-CU-NEXT:    s_clause 0x1
11374; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11375; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11376; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11377; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11378; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11379; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11380; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11381; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11382; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11383; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11384; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11385; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11386; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11387; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11388; GFX10-CU-NEXT:    s_endpgm
11389;
11390; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11391; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11392; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11393; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11394; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11395; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11396; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11397; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11398; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11399; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11400; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11401; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11402; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11403; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11404; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11405; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11406; SKIP-CACHE-INV-NEXT:    s_endpgm
11407;
11408; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11409; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11410; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11411; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11412; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11413; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11414; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11415; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11416; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11417; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11418; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11419;
11420; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11421; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11422; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11423; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11424; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11425; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11426; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11427; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11428; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11429; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11430; GFX90A-TGSPLIT-NEXT:    s_endpgm
11431;
11432; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11433; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11434; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11435; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11436; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11437; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11438; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11439; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11440; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11441; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11442; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11443;
11444; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11445; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11446; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11447; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11448; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11449; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11450; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11451; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11452; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11453; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11454; GFX940-TGSPLIT-NEXT:    s_endpgm
11455;
11456; GFX11-WGP-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11457; GFX11-WGP:       ; %bb.0: ; %entry
11458; GFX11-WGP-NEXT:    s_clause 0x1
11459; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11460; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11461; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11462; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11463; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11464; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11465; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11466; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11467; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11468; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11469; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11470; GFX11-WGP-NEXT:    s_endpgm
11471;
11472; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11473; GFX11-CU:       ; %bb.0: ; %entry
11474; GFX11-CU-NEXT:    s_clause 0x1
11475; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11476; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11477; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11478; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11479; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11480; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11481; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11482; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11483; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11484; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11485; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11486; GFX11-CU-NEXT:    s_endpgm
11487    i32* %out, i32 %in, i32 %old) {
11488entry:
11489  %gep = getelementptr i32, i32* %out, i32 4
11490  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic acquire
11491  %val0 = extractvalue { i32, i1 } %val, 0
11492  store i32 %val0, i32* %out, align 4
11493  ret void
11494}
11495
11496define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_ret_cmpxchg(
11497; GFX7-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11498; GFX7:       ; %bb.0: ; %entry
11499; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11500; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11501; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11502; GFX7-NEXT:    s_add_u32 s4, s0, 16
11503; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11504; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11505; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11506; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11507; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11508; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11509; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11510; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11511; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11512; GFX7-NEXT:    flat_store_dword v[0:1], v2
11513; GFX7-NEXT:    s_endpgm
11514;
11515; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11516; GFX10-WGP:       ; %bb.0: ; %entry
11517; GFX10-WGP-NEXT:    s_clause 0x1
11518; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11519; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11520; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11521; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11522; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11523; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11524; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11525; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11526; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11527; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11528; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11529; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11530; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11531; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11532; GFX10-WGP-NEXT:    s_endpgm
11533;
11534; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11535; GFX10-CU:       ; %bb.0: ; %entry
11536; GFX10-CU-NEXT:    s_clause 0x1
11537; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11538; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11539; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11540; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11541; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11542; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11543; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11544; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11545; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11546; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11547; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11548; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11549; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11550; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11551; GFX10-CU-NEXT:    s_endpgm
11552;
11553; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11554; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11555; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11556; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11557; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11558; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11559; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11560; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11561; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11562; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11563; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11564; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11565; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11566; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11567; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11568; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11569; SKIP-CACHE-INV-NEXT:    s_endpgm
11570;
11571; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11572; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11573; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11574; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11575; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11576; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11577; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11578; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11579; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11580; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11581; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11582;
11583; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11584; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11585; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11586; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11587; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11588; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11589; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11590; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11591; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11592; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11593; GFX90A-TGSPLIT-NEXT:    s_endpgm
11594;
11595; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11596; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11597; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11598; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11599; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11600; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11601; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11602; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11603; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11604; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11605; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11606;
11607; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11608; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11609; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11610; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11611; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11612; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11613; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11614; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11615; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11616; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11617; GFX940-TGSPLIT-NEXT:    s_endpgm
11618;
11619; GFX11-WGP-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11620; GFX11-WGP:       ; %bb.0: ; %entry
11621; GFX11-WGP-NEXT:    s_clause 0x1
11622; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11623; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11624; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11625; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11626; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11627; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11628; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11629; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11630; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11631; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11632; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11633; GFX11-WGP-NEXT:    s_endpgm
11634;
11635; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11636; GFX11-CU:       ; %bb.0: ; %entry
11637; GFX11-CU-NEXT:    s_clause 0x1
11638; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11639; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11640; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11641; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11642; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11643; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11644; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11645; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11646; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11647; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11648; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11649; GFX11-CU-NEXT:    s_endpgm
11650    i32* %out, i32 %in, i32 %old) {
11651entry:
11652  %gep = getelementptr i32, i32* %out, i32 4
11653  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
11654  %val0 = extractvalue { i32, i1 } %val, 0
11655  store i32 %val0, i32* %out, align 4
11656  ret void
11657}
11658
11659define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_ret_cmpxchg(
11660; GFX7-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
11661; GFX7:       ; %bb.0: ; %entry
11662; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11663; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11664; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11665; GFX7-NEXT:    s_add_u32 s4, s0, 16
11666; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11667; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11668; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11669; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11670; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11671; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11672; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11673; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11674; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11675; GFX7-NEXT:    flat_store_dword v[0:1], v2
11676; GFX7-NEXT:    s_endpgm
11677;
11678; GFX10-WGP-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
11679; GFX10-WGP:       ; %bb.0: ; %entry
11680; GFX10-WGP-NEXT:    s_clause 0x1
11681; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11682; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11683; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11684; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11685; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11686; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11687; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11688; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11689; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11690; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11691; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11692; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11693; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11694; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11695; GFX10-WGP-NEXT:    s_endpgm
11696;
11697; GFX10-CU-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
11698; GFX10-CU:       ; %bb.0: ; %entry
11699; GFX10-CU-NEXT:    s_clause 0x1
11700; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11701; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11702; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11703; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11704; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11705; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11706; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11707; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11708; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11709; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11710; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11711; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11712; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11713; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11714; GFX10-CU-NEXT:    s_endpgm
11715;
11716; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
11717; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11718; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11719; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11720; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11721; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11722; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11723; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11724; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11725; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11726; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11727; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11728; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11729; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11730; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11731; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11732; SKIP-CACHE-INV-NEXT:    s_endpgm
11733;
11734; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
11735; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11736; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11737; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11738; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11739; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11740; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11741; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11742; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11743; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11744; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11745;
11746; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
11747; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11748; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11749; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11750; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11751; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11752; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11753; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11754; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11755; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11756; GFX90A-TGSPLIT-NEXT:    s_endpgm
11757;
11758; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
11759; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11760; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11761; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11762; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11763; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11764; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11765; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11766; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11767; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11768; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11769;
11770; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
11771; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11772; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11773; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11774; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11775; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11776; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11777; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11778; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11779; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11780; GFX940-TGSPLIT-NEXT:    s_endpgm
11781;
11782; GFX11-WGP-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
11783; GFX11-WGP:       ; %bb.0: ; %entry
11784; GFX11-WGP-NEXT:    s_clause 0x1
11785; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11786; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11787; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11788; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11789; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11790; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11791; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11792; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11793; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11794; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11795; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11796; GFX11-WGP-NEXT:    s_endpgm
11797;
11798; GFX11-CU-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
11799; GFX11-CU:       ; %bb.0: ; %entry
11800; GFX11-CU-NEXT:    s_clause 0x1
11801; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11802; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11803; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11804; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11805; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11806; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11807; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11808; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11809; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11810; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11811; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11812; GFX11-CU-NEXT:    s_endpgm
11813    i32* %out, i32 %in, i32 %old) {
11814entry:
11815  %gep = getelementptr i32, i32* %out, i32 4
11816  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
11817  %val0 = extractvalue { i32, i1 } %val, 0
11818  store i32 %val0, i32* %out, align 4
11819  ret void
11820}
11821
11822define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
11823; GFX7-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11824; GFX7:       ; %bb.0: ; %entry
11825; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11826; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11827; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11828; GFX7-NEXT:    s_add_u32 s4, s0, 16
11829; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11830; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11831; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11832; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11833; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11834; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11835; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11836; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11837; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11838; GFX7-NEXT:    flat_store_dword v[0:1], v2
11839; GFX7-NEXT:    s_endpgm
11840;
11841; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11842; GFX10-WGP:       ; %bb.0: ; %entry
11843; GFX10-WGP-NEXT:    s_clause 0x1
11844; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11845; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11846; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11847; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11848; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11849; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11850; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11851; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11852; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11853; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11854; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11855; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11856; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11857; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11858; GFX10-WGP-NEXT:    s_endpgm
11859;
11860; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11861; GFX10-CU:       ; %bb.0: ; %entry
11862; GFX10-CU-NEXT:    s_clause 0x1
11863; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11864; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11865; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11866; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11867; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11868; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11869; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11870; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11871; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11872; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11873; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11874; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11875; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11876; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11877; GFX10-CU-NEXT:    s_endpgm
11878;
11879; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11880; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11881; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11882; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11883; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11884; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11885; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11886; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11887; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11888; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11889; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11890; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11891; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11892; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11893; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11894; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11895; SKIP-CACHE-INV-NEXT:    s_endpgm
11896;
11897; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11898; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11899; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11900; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11901; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11902; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11903; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11904; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11905; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11906; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11907; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11908;
11909; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11910; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11911; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11912; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11913; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11914; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11915; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11916; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11917; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11918; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11919; GFX90A-TGSPLIT-NEXT:    s_endpgm
11920;
11921; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11922; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11923; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11924; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11925; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11926; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11927; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11928; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11929; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11930; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11931; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11932;
11933; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11934; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11935; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11936; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11937; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11938; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11939; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11940; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11941; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11942; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11943; GFX940-TGSPLIT-NEXT:    s_endpgm
11944;
11945; GFX11-WGP-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11946; GFX11-WGP:       ; %bb.0: ; %entry
11947; GFX11-WGP-NEXT:    s_clause 0x1
11948; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11949; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11950; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11951; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11952; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11953; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11954; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11955; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11956; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11957; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11958; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11959; GFX11-WGP-NEXT:    s_endpgm
11960;
11961; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11962; GFX11-CU:       ; %bb.0: ; %entry
11963; GFX11-CU-NEXT:    s_clause 0x1
11964; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11965; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11966; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11967; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11968; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11969; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11970; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11971; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11972; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11973; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11974; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11975; GFX11-CU-NEXT:    s_endpgm
11976    i32* %out, i32 %in, i32 %old) {
11977entry:
11978  %gep = getelementptr i32, i32* %out, i32 4
11979  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
11980  %val0 = extractvalue { i32, i1 } %val, 0
11981  store i32 %val0, i32* %out, align 4
11982  ret void
11983}
11984
11985define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
11986; GFX7-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11987; GFX7:       ; %bb.0: ; %entry
11988; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11989; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11990; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11991; GFX7-NEXT:    s_add_u32 s4, s0, 16
11992; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11993; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11994; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11995; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11996; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11997; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11998; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11999; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12000; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12001; GFX7-NEXT:    flat_store_dword v[0:1], v2
12002; GFX7-NEXT:    s_endpgm
12003;
12004; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12005; GFX10-WGP:       ; %bb.0: ; %entry
12006; GFX10-WGP-NEXT:    s_clause 0x1
12007; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12008; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12009; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12010; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12011; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12012; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12013; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12014; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12015; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12016; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12017; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12018; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12019; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12020; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12021; GFX10-WGP-NEXT:    s_endpgm
12022;
12023; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12024; GFX10-CU:       ; %bb.0: ; %entry
12025; GFX10-CU-NEXT:    s_clause 0x1
12026; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12027; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12028; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12029; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12030; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12031; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12032; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12033; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12034; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12035; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12036; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12037; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12038; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12039; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12040; GFX10-CU-NEXT:    s_endpgm
12041;
12042; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12043; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12044; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12045; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12046; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12047; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12048; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12049; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12050; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12051; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12052; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12053; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12054; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12055; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12056; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12057; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12058; SKIP-CACHE-INV-NEXT:    s_endpgm
12059;
12060; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12061; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12062; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12063; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12064; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12065; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12066; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12067; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12068; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12069; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12070; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12071;
12072; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12073; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12074; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12075; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12076; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12077; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12078; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12079; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12080; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12081; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12082; GFX90A-TGSPLIT-NEXT:    s_endpgm
12083;
12084; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12085; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12086; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12087; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12088; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12089; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12090; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12091; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12092; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12093; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12094; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12095;
12096; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12097; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12098; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12099; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12100; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12101; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12102; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12103; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12104; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12105; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12106; GFX940-TGSPLIT-NEXT:    s_endpgm
12107;
12108; GFX11-WGP-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12109; GFX11-WGP:       ; %bb.0: ; %entry
12110; GFX11-WGP-NEXT:    s_clause 0x1
12111; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12112; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12113; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12114; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12115; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12116; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12117; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12118; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12119; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12120; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12121; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12122; GFX11-WGP-NEXT:    s_endpgm
12123;
12124; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12125; GFX11-CU:       ; %bb.0: ; %entry
12126; GFX11-CU-NEXT:    s_clause 0x1
12127; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12128; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12129; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12130; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12131; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12132; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12133; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12134; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12135; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12136; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12137; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12138; GFX11-CU-NEXT:    s_endpgm
12139    i32* %out, i32 %in, i32 %old) {
12140entry:
12141  %gep = getelementptr i32, i32* %out, i32 4
12142  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
12143  %val0 = extractvalue { i32, i1 } %val, 0
12144  store i32 %val0, i32* %out, align 4
12145  ret void
12146}
12147
12148define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg(
12149; GFX7-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12150; GFX7:       ; %bb.0: ; %entry
12151; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12152; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12153; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12154; GFX7-NEXT:    s_add_u32 s4, s0, 16
12155; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12156; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12157; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12158; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12159; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12160; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12161; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12162; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12163; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12164; GFX7-NEXT:    flat_store_dword v[0:1], v2
12165; GFX7-NEXT:    s_endpgm
12166;
12167; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12168; GFX10-WGP:       ; %bb.0: ; %entry
12169; GFX10-WGP-NEXT:    s_clause 0x1
12170; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12171; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12172; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12173; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12174; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12175; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12176; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12177; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12178; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12179; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12180; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12181; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12182; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12183; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12184; GFX10-WGP-NEXT:    s_endpgm
12185;
12186; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12187; GFX10-CU:       ; %bb.0: ; %entry
12188; GFX10-CU-NEXT:    s_clause 0x1
12189; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12190; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12191; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12192; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12193; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12194; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12195; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12196; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12197; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12198; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12199; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12200; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12201; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12202; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12203; GFX10-CU-NEXT:    s_endpgm
12204;
12205; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12206; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12207; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12208; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12209; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12210; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12211; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12212; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12213; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12214; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12215; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12216; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12217; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12218; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12219; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12220; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12221; SKIP-CACHE-INV-NEXT:    s_endpgm
12222;
12223; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12224; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12225; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12226; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12227; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12228; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12229; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12230; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12231; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12232; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12233; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12234;
12235; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12236; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12237; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12238; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12239; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12240; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12241; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12242; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12243; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12244; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12245; GFX90A-TGSPLIT-NEXT:    s_endpgm
12246;
12247; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12248; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12249; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12250; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12251; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12252; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12253; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12254; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12255; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12256; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12257; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12258;
12259; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12260; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12261; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12262; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12263; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12264; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12265; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12266; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12267; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12268; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12269; GFX940-TGSPLIT-NEXT:    s_endpgm
12270;
12271; GFX11-WGP-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12272; GFX11-WGP:       ; %bb.0: ; %entry
12273; GFX11-WGP-NEXT:    s_clause 0x1
12274; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12275; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12276; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12277; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12278; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12279; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12280; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12281; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12282; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12283; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12284; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12285; GFX11-WGP-NEXT:    s_endpgm
12286;
12287; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12288; GFX11-CU:       ; %bb.0: ; %entry
12289; GFX11-CU-NEXT:    s_clause 0x1
12290; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12291; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12292; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12293; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12294; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12295; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12296; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12297; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12298; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12299; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12300; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12301; GFX11-CU-NEXT:    s_endpgm
12302    i32* %out, i32 %in, i32 %old) {
12303entry:
12304  %gep = getelementptr i32, i32* %out, i32 4
12305  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic seq_cst
12306  %val0 = extractvalue { i32, i1 } %val, 0
12307  store i32 %val0, i32* %out, align 4
12308  ret void
12309}
12310
12311define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg(
12312; GFX7-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12313; GFX7:       ; %bb.0: ; %entry
12314; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12315; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12316; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12317; GFX7-NEXT:    s_add_u32 s4, s0, 16
12318; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12319; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12320; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12321; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12322; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12323; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12324; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12325; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12326; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12327; GFX7-NEXT:    flat_store_dword v[0:1], v2
12328; GFX7-NEXT:    s_endpgm
12329;
12330; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12331; GFX10-WGP:       ; %bb.0: ; %entry
12332; GFX10-WGP-NEXT:    s_clause 0x1
12333; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12334; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12335; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12336; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12337; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12338; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12339; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12340; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12341; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12342; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12343; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12344; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12345; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12346; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12347; GFX10-WGP-NEXT:    s_endpgm
12348;
12349; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12350; GFX10-CU:       ; %bb.0: ; %entry
12351; GFX10-CU-NEXT:    s_clause 0x1
12352; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12353; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12354; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12355; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12356; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12357; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12358; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12359; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12360; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12361; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12362; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12363; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12364; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12365; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12366; GFX10-CU-NEXT:    s_endpgm
12367;
12368; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12369; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12370; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12371; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12372; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12373; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12374; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12375; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12376; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12377; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12378; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12379; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12380; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12381; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12382; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12383; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12384; SKIP-CACHE-INV-NEXT:    s_endpgm
12385;
12386; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12387; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12388; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12389; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12390; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12391; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12392; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12393; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12394; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12395; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12396; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12397;
12398; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12399; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12400; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12401; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12402; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12403; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12404; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12405; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12406; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12407; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12408; GFX90A-TGSPLIT-NEXT:    s_endpgm
12409;
12410; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12411; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12412; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12413; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12414; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12415; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12416; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12417; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12418; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12419; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12420; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12421;
12422; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12423; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12424; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12425; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12426; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12427; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12428; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12429; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12430; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12431; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12432; GFX940-TGSPLIT-NEXT:    s_endpgm
12433;
12434; GFX11-WGP-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12435; GFX11-WGP:       ; %bb.0: ; %entry
12436; GFX11-WGP-NEXT:    s_clause 0x1
12437; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12438; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12439; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12440; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12441; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12442; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12443; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12444; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12445; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12446; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12447; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12448; GFX11-WGP-NEXT:    s_endpgm
12449;
12450; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12451; GFX11-CU:       ; %bb.0: ; %entry
12452; GFX11-CU-NEXT:    s_clause 0x1
12453; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12454; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12455; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12456; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12457; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12458; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12459; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12460; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12461; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12462; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12463; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12464; GFX11-CU-NEXT:    s_endpgm
12465    i32* %out, i32 %in, i32 %old) {
12466entry:
12467  %gep = getelementptr i32, i32* %out, i32 4
12468  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire seq_cst
12469  %val0 = extractvalue { i32, i1 } %val, 0
12470  store i32 %val0, i32* %out, align 4
12471  ret void
12472}
12473
12474define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_ret_cmpxchg(
12475; GFX7-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12476; GFX7:       ; %bb.0: ; %entry
12477; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12478; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12479; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12480; GFX7-NEXT:    s_add_u32 s4, s0, 16
12481; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12482; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12483; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12484; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12485; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12486; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12487; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12488; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12489; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12490; GFX7-NEXT:    flat_store_dword v[0:1], v2
12491; GFX7-NEXT:    s_endpgm
12492;
12493; GFX10-WGP-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12494; GFX10-WGP:       ; %bb.0: ; %entry
12495; GFX10-WGP-NEXT:    s_clause 0x1
12496; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12497; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12498; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12499; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12500; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12501; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12502; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12503; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12504; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12505; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12506; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12507; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12508; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12509; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12510; GFX10-WGP-NEXT:    s_endpgm
12511;
12512; GFX10-CU-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12513; GFX10-CU:       ; %bb.0: ; %entry
12514; GFX10-CU-NEXT:    s_clause 0x1
12515; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12516; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12517; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12518; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12519; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12520; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12521; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12522; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12523; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12524; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12525; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12526; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12527; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12528; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12529; GFX10-CU-NEXT:    s_endpgm
12530;
12531; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12532; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12533; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12534; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12535; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12536; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12537; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12538; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12539; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12540; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12541; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12542; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12543; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12544; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12545; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12546; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12547; SKIP-CACHE-INV-NEXT:    s_endpgm
12548;
12549; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12550; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12551; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12552; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12553; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12554; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12555; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12556; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12557; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12558; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12559; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12560;
12561; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12562; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12563; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12564; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12565; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12566; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12567; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12568; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12569; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12570; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12571; GFX90A-TGSPLIT-NEXT:    s_endpgm
12572;
12573; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12574; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12575; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12576; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12577; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12578; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12579; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12580; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12581; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12582; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12583; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12584;
12585; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12586; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12587; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12588; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12589; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12590; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12591; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12592; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12593; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12594; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12595; GFX940-TGSPLIT-NEXT:    s_endpgm
12596;
12597; GFX11-WGP-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12598; GFX11-WGP:       ; %bb.0: ; %entry
12599; GFX11-WGP-NEXT:    s_clause 0x1
12600; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12601; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12602; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12603; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12604; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12605; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12606; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12607; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12608; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12609; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12610; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12611; GFX11-WGP-NEXT:    s_endpgm
12612;
12613; GFX11-CU-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12614; GFX11-CU:       ; %bb.0: ; %entry
12615; GFX11-CU-NEXT:    s_clause 0x1
12616; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12617; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12618; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12619; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12620; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12621; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12622; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12623; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12624; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12625; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12626; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12627; GFX11-CU-NEXT:    s_endpgm
12628    i32* %out, i32 %in, i32 %old) {
12629entry:
12630  %gep = getelementptr i32, i32* %out, i32 4
12631  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release seq_cst
12632  %val0 = extractvalue { i32, i1 } %val, 0
12633  store i32 %val0, i32* %out, align 4
12634  ret void
12635}
12636
12637define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg(
12638; GFX7-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12639; GFX7:       ; %bb.0: ; %entry
12640; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12641; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12642; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12643; GFX7-NEXT:    s_add_u32 s4, s0, 16
12644; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12645; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12646; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12647; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12648; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12649; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12650; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12651; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12652; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12653; GFX7-NEXT:    flat_store_dword v[0:1], v2
12654; GFX7-NEXT:    s_endpgm
12655;
12656; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12657; GFX10-WGP:       ; %bb.0: ; %entry
12658; GFX10-WGP-NEXT:    s_clause 0x1
12659; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12660; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12661; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12662; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12663; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12664; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12665; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12666; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12667; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12668; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12669; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12670; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12671; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12672; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12673; GFX10-WGP-NEXT:    s_endpgm
12674;
12675; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12676; GFX10-CU:       ; %bb.0: ; %entry
12677; GFX10-CU-NEXT:    s_clause 0x1
12678; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12679; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12680; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12681; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12682; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12683; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12684; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12685; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12686; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12687; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12688; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12689; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12690; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12691; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12692; GFX10-CU-NEXT:    s_endpgm
12693;
12694; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12695; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12696; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12697; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12698; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12699; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12700; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12701; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12702; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12703; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12704; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12705; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12706; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12707; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12708; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12709; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12710; SKIP-CACHE-INV-NEXT:    s_endpgm
12711;
12712; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12713; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12714; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12715; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12716; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12717; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12718; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12719; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12720; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12721; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12722; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12723;
12724; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12725; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12726; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12727; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12728; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12729; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12730; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12731; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12732; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12733; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12734; GFX90A-TGSPLIT-NEXT:    s_endpgm
12735;
12736; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12737; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12738; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12739; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12740; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12741; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12742; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12743; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12744; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12745; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12746; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12747;
12748; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12749; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12750; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12751; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12752; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12753; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12754; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12755; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12756; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12757; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12758; GFX940-TGSPLIT-NEXT:    s_endpgm
12759;
12760; GFX11-WGP-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12761; GFX11-WGP:       ; %bb.0: ; %entry
12762; GFX11-WGP-NEXT:    s_clause 0x1
12763; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12764; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12765; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12766; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12767; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12768; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12769; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12770; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12771; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12772; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12773; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12774; GFX11-WGP-NEXT:    s_endpgm
12775;
12776; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12777; GFX11-CU:       ; %bb.0: ; %entry
12778; GFX11-CU-NEXT:    s_clause 0x1
12779; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12780; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12781; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12782; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12783; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12784; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12785; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12786; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12787; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12788; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12789; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12790; GFX11-CU-NEXT:    s_endpgm
12791    i32* %out, i32 %in, i32 %old) {
12792entry:
12793  %gep = getelementptr i32, i32* %out, i32 4
12794  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel seq_cst
12795  %val0 = extractvalue { i32, i1 } %val, 0
12796  store i32 %val0, i32* %out, align 4
12797  ret void
12798}
12799
12800define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
12801; GFX7-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12802; GFX7:       ; %bb.0: ; %entry
12803; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12804; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12805; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12806; GFX7-NEXT:    s_add_u32 s4, s0, 16
12807; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12808; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12809; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12810; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12811; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12812; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12813; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12814; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12815; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12816; GFX7-NEXT:    flat_store_dword v[0:1], v2
12817; GFX7-NEXT:    s_endpgm
12818;
12819; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12820; GFX10-WGP:       ; %bb.0: ; %entry
12821; GFX10-WGP-NEXT:    s_clause 0x1
12822; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12823; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12824; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12825; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12826; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12827; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12828; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12829; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12830; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12831; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12832; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12833; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12834; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12835; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12836; GFX10-WGP-NEXT:    s_endpgm
12837;
12838; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12839; GFX10-CU:       ; %bb.0: ; %entry
12840; GFX10-CU-NEXT:    s_clause 0x1
12841; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12842; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12843; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12844; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12845; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12846; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12847; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12848; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12849; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12850; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12851; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12852; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12853; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12854; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12855; GFX10-CU-NEXT:    s_endpgm
12856;
12857; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12858; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12859; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12860; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12861; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12862; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12863; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12864; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12865; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12866; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12867; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12868; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12869; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12870; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12871; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12872; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12873; SKIP-CACHE-INV-NEXT:    s_endpgm
12874;
12875; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12876; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12877; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12878; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12879; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12880; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12881; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12882; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12883; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12884; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12885; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12886;
12887; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12888; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12889; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12890; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12891; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12892; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12893; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12894; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12895; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12896; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12897; GFX90A-TGSPLIT-NEXT:    s_endpgm
12898;
12899; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12900; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12901; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12902; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12903; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12904; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12905; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12906; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12907; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12908; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12909; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12910;
12911; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12912; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12913; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12914; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12915; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12916; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12917; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12918; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12919; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12920; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12921; GFX940-TGSPLIT-NEXT:    s_endpgm
12922;
12923; GFX11-WGP-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12924; GFX11-WGP:       ; %bb.0: ; %entry
12925; GFX11-WGP-NEXT:    s_clause 0x1
12926; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12927; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12928; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12929; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12930; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12931; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12932; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12933; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12934; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12935; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12936; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12937; GFX11-WGP-NEXT:    s_endpgm
12938;
12939; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12940; GFX11-CU:       ; %bb.0: ; %entry
12941; GFX11-CU-NEXT:    s_clause 0x1
12942; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12943; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12944; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12945; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12946; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12947; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12948; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12949; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12950; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12951; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12952; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12953; GFX11-CU-NEXT:    s_endpgm
12954    i32* %out, i32 %in, i32 %old) {
12955entry:
12956  %gep = getelementptr i32, i32* %out, i32 4
12957  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
12958  %val0 = extractvalue { i32, i1 } %val, 0
12959  store i32 %val0, i32* %out, align 4
12960  ret void
12961}
12962
12963