1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
8; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
9; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
10; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-WGP %s
11; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-CU %s
12
13define amdgpu_kernel void @flat_singlethread_unordered_load(
14; GFX7-LABEL: flat_singlethread_unordered_load:
15; GFX7:       ; %bb.0: ; %entry
16; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
17; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
18; GFX7-NEXT:    v_mov_b32_e32 v0, s0
19; GFX7-NEXT:    v_mov_b32_e32 v1, s1
20; GFX7-NEXT:    flat_load_dword v2, v[0:1]
21; GFX7-NEXT:    v_mov_b32_e32 v0, s2
22; GFX7-NEXT:    v_mov_b32_e32 v1, s3
23; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
24; GFX7-NEXT:    flat_store_dword v[0:1], v2
25; GFX7-NEXT:    s_endpgm
26;
27; GFX10-WGP-LABEL: flat_singlethread_unordered_load:
28; GFX10-WGP:       ; %bb.0: ; %entry
29; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
30; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
31; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
32; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
33; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
34; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
35; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
36; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
37; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
38; GFX10-WGP-NEXT:    s_endpgm
39;
40; GFX10-CU-LABEL: flat_singlethread_unordered_load:
41; GFX10-CU:       ; %bb.0: ; %entry
42; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
43; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
44; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
45; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
46; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
47; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
48; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
49; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
50; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
51; GFX10-CU-NEXT:    s_endpgm
52;
53; SKIP-CACHE-INV-LABEL: flat_singlethread_unordered_load:
54; SKIP-CACHE-INV:       ; %bb.0: ; %entry
55; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
56; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
57; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
58; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
59; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
60; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
61; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
62; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
63; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
64; SKIP-CACHE-INV-NEXT:    s_endpgm
65;
66; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_unordered_load:
67; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
68; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
69; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
70; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
71; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
72; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
73; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
74; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
75; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
76; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
77; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
78;
79; GFX90A-TGSPLIT-LABEL: flat_singlethread_unordered_load:
80; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
81; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
82; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
83; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
84; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
85; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
86; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
87; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
88; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
89; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
90; GFX90A-TGSPLIT-NEXT:    s_endpgm
91;
92; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_unordered_load:
93; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
94; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
95; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
96; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
97; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
98; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
99; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
100; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
101; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
102; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
103; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
104;
105; GFX940-TGSPLIT-LABEL: flat_singlethread_unordered_load:
106; GFX940-TGSPLIT:       ; %bb.0: ; %entry
107; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
108; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
109; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
110; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
111; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
112; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
113; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
114; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
115; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
116; GFX940-TGSPLIT-NEXT:    s_endpgm
117;
118; GFX11-WGP-LABEL: flat_singlethread_unordered_load:
119; GFX11-WGP:       ; %bb.0: ; %entry
120; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
121; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
122; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
123; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
124; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
125; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
126; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
127; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
128; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
129; GFX11-WGP-NEXT:    s_endpgm
130;
131; GFX11-CU-LABEL: flat_singlethread_unordered_load:
132; GFX11-CU:       ; %bb.0: ; %entry
133; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
134; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
135; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
136; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
137; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
138; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
139; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
140; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
141; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
142; GFX11-CU-NEXT:    s_endpgm
143    i32* %in, i32* %out) {
144entry:
145  %val = load atomic i32, i32* %in syncscope("singlethread") unordered, align 4
146  store i32 %val, i32* %out
147  ret void
148}
149
150define amdgpu_kernel void @flat_singlethread_monotonic_load(
151; GFX7-LABEL: flat_singlethread_monotonic_load:
152; GFX7:       ; %bb.0: ; %entry
153; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
154; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
155; GFX7-NEXT:    v_mov_b32_e32 v0, s0
156; GFX7-NEXT:    v_mov_b32_e32 v1, s1
157; GFX7-NEXT:    flat_load_dword v2, v[0:1]
158; GFX7-NEXT:    v_mov_b32_e32 v0, s2
159; GFX7-NEXT:    v_mov_b32_e32 v1, s3
160; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
161; GFX7-NEXT:    flat_store_dword v[0:1], v2
162; GFX7-NEXT:    s_endpgm
163;
164; GFX10-WGP-LABEL: flat_singlethread_monotonic_load:
165; GFX10-WGP:       ; %bb.0: ; %entry
166; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
167; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
168; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
169; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
170; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
171; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
172; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
173; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
174; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
175; GFX10-WGP-NEXT:    s_endpgm
176;
177; GFX10-CU-LABEL: flat_singlethread_monotonic_load:
178; GFX10-CU:       ; %bb.0: ; %entry
179; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
180; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
181; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
182; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
183; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
184; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
185; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
186; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
187; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
188; GFX10-CU-NEXT:    s_endpgm
189;
190; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_load:
191; SKIP-CACHE-INV:       ; %bb.0: ; %entry
192; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
193; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
194; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
195; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
196; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
197; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
198; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
199; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
200; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
201; SKIP-CACHE-INV-NEXT:    s_endpgm
202;
203; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_load:
204; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
205; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
206; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
207; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
208; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
209; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
210; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
211; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
212; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
213; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
214; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
215;
216; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_load:
217; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
218; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
219; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
220; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
221; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
222; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
223; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
224; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
225; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
226; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
227; GFX90A-TGSPLIT-NEXT:    s_endpgm
228;
229; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_load:
230; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
231; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
232; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
233; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
234; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
235; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
236; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
237; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
238; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
239; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
240; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
241;
242; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_load:
243; GFX940-TGSPLIT:       ; %bb.0: ; %entry
244; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
245; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
246; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
247; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
248; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
249; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
250; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
251; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
252; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
253; GFX940-TGSPLIT-NEXT:    s_endpgm
254;
255; GFX11-WGP-LABEL: flat_singlethread_monotonic_load:
256; GFX11-WGP:       ; %bb.0: ; %entry
257; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
258; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
259; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
260; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
261; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
262; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
263; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
264; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
265; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
266; GFX11-WGP-NEXT:    s_endpgm
267;
268; GFX11-CU-LABEL: flat_singlethread_monotonic_load:
269; GFX11-CU:       ; %bb.0: ; %entry
270; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
271; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
272; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
273; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
274; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
275; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
276; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
277; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
278; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
279; GFX11-CU-NEXT:    s_endpgm
280    i32* %in, i32* %out) {
281entry:
282  %val = load atomic i32, i32* %in syncscope("singlethread") monotonic, align 4
283  store i32 %val, i32* %out
284  ret void
285}
286
287define amdgpu_kernel void @flat_singlethread_acquire_load(
288; GFX7-LABEL: flat_singlethread_acquire_load:
289; GFX7:       ; %bb.0: ; %entry
290; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
291; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
292; GFX7-NEXT:    v_mov_b32_e32 v0, s0
293; GFX7-NEXT:    v_mov_b32_e32 v1, s1
294; GFX7-NEXT:    flat_load_dword v2, v[0:1]
295; GFX7-NEXT:    v_mov_b32_e32 v0, s2
296; GFX7-NEXT:    v_mov_b32_e32 v1, s3
297; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
298; GFX7-NEXT:    flat_store_dword v[0:1], v2
299; GFX7-NEXT:    s_endpgm
300;
301; GFX10-WGP-LABEL: flat_singlethread_acquire_load:
302; GFX10-WGP:       ; %bb.0: ; %entry
303; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
304; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
305; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
306; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
307; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
308; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
309; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
310; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
311; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
312; GFX10-WGP-NEXT:    s_endpgm
313;
314; GFX10-CU-LABEL: flat_singlethread_acquire_load:
315; GFX10-CU:       ; %bb.0: ; %entry
316; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
317; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
318; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
319; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
320; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
321; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
322; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
323; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
324; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
325; GFX10-CU-NEXT:    s_endpgm
326;
327; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_load:
328; SKIP-CACHE-INV:       ; %bb.0: ; %entry
329; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
330; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
331; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
332; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
333; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
334; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
335; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
336; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
337; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
338; SKIP-CACHE-INV-NEXT:    s_endpgm
339;
340; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_load:
341; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
342; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
343; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
344; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
345; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
346; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
347; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
348; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
349; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
350; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
351; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
352;
353; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_load:
354; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
355; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
356; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
357; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
358; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
359; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
360; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
361; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
362; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
363; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
364; GFX90A-TGSPLIT-NEXT:    s_endpgm
365;
366; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_load:
367; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
368; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
369; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
370; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
371; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
372; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
373; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
374; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
375; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
376; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
377; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
378;
379; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_load:
380; GFX940-TGSPLIT:       ; %bb.0: ; %entry
381; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
382; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
383; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
384; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
385; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
386; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
387; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
388; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
389; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
390; GFX940-TGSPLIT-NEXT:    s_endpgm
391;
392; GFX11-WGP-LABEL: flat_singlethread_acquire_load:
393; GFX11-WGP:       ; %bb.0: ; %entry
394; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
395; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
396; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
397; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
398; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
399; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
400; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
401; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
402; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
403; GFX11-WGP-NEXT:    s_endpgm
404;
405; GFX11-CU-LABEL: flat_singlethread_acquire_load:
406; GFX11-CU:       ; %bb.0: ; %entry
407; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
408; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
409; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
410; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
411; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
412; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
413; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
414; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
415; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
416; GFX11-CU-NEXT:    s_endpgm
417    i32* %in, i32* %out) {
418entry:
419  %val = load atomic i32, i32* %in syncscope("singlethread") acquire, align 4
420  store i32 %val, i32* %out
421  ret void
422}
423
424define amdgpu_kernel void @flat_singlethread_seq_cst_load(
425; GFX7-LABEL: flat_singlethread_seq_cst_load:
426; GFX7:       ; %bb.0: ; %entry
427; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
428; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
429; GFX7-NEXT:    v_mov_b32_e32 v0, s0
430; GFX7-NEXT:    v_mov_b32_e32 v1, s1
431; GFX7-NEXT:    flat_load_dword v2, v[0:1]
432; GFX7-NEXT:    v_mov_b32_e32 v0, s2
433; GFX7-NEXT:    v_mov_b32_e32 v1, s3
434; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
435; GFX7-NEXT:    flat_store_dword v[0:1], v2
436; GFX7-NEXT:    s_endpgm
437;
438; GFX10-WGP-LABEL: flat_singlethread_seq_cst_load:
439; GFX10-WGP:       ; %bb.0: ; %entry
440; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
441; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
442; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
443; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
444; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
445; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
446; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
447; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
448; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
449; GFX10-WGP-NEXT:    s_endpgm
450;
451; GFX10-CU-LABEL: flat_singlethread_seq_cst_load:
452; GFX10-CU:       ; %bb.0: ; %entry
453; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
454; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
455; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
456; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
457; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
458; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
459; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
460; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
461; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
462; GFX10-CU-NEXT:    s_endpgm
463;
464; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_load:
465; SKIP-CACHE-INV:       ; %bb.0: ; %entry
466; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
467; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
468; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
469; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
470; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
471; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
472; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
473; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
474; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
475; SKIP-CACHE-INV-NEXT:    s_endpgm
476;
477; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_load:
478; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
479; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
480; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
481; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
482; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
483; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
484; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
485; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
486; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
487; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
488; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
489;
490; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_load:
491; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
492; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
493; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
494; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
495; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
496; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
497; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
498; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
499; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
500; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
501; GFX90A-TGSPLIT-NEXT:    s_endpgm
502;
503; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_load:
504; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
505; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
506; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
507; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
508; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
509; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
510; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
511; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
512; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
513; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
514; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
515;
516; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_load:
517; GFX940-TGSPLIT:       ; %bb.0: ; %entry
518; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
519; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
520; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
521; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
522; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
523; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
524; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
525; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
526; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
527; GFX940-TGSPLIT-NEXT:    s_endpgm
528;
529; GFX11-WGP-LABEL: flat_singlethread_seq_cst_load:
530; GFX11-WGP:       ; %bb.0: ; %entry
531; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
532; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
533; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
534; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
535; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
536; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
537; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
538; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
539; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
540; GFX11-WGP-NEXT:    s_endpgm
541;
542; GFX11-CU-LABEL: flat_singlethread_seq_cst_load:
543; GFX11-CU:       ; %bb.0: ; %entry
544; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
545; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
546; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
547; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
548; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
549; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
550; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
551; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
552; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
553; GFX11-CU-NEXT:    s_endpgm
554    i32* %in, i32* %out) {
555entry:
556  %val = load atomic i32, i32* %in syncscope("singlethread") seq_cst, align 4
557  store i32 %val, i32* %out
558  ret void
559}
560
561define amdgpu_kernel void @flat_singlethread_unordered_store(
562; GFX7-LABEL: flat_singlethread_unordered_store:
563; GFX7:       ; %bb.0: ; %entry
564; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
565; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
566; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
567; GFX7-NEXT:    v_mov_b32_e32 v0, s0
568; GFX7-NEXT:    v_mov_b32_e32 v1, s1
569; GFX7-NEXT:    v_mov_b32_e32 v2, s2
570; GFX7-NEXT:    flat_store_dword v[0:1], v2
571; GFX7-NEXT:    s_endpgm
572;
573; GFX10-WGP-LABEL: flat_singlethread_unordered_store:
574; GFX10-WGP:       ; %bb.0: ; %entry
575; GFX10-WGP-NEXT:    s_clause 0x1
576; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
577; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
578; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
579; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
580; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
581; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
582; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
583; GFX10-WGP-NEXT:    s_endpgm
584;
585; GFX10-CU-LABEL: flat_singlethread_unordered_store:
586; GFX10-CU:       ; %bb.0: ; %entry
587; GFX10-CU-NEXT:    s_clause 0x1
588; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
589; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
590; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
591; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
592; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
593; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
594; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
595; GFX10-CU-NEXT:    s_endpgm
596;
597; SKIP-CACHE-INV-LABEL: flat_singlethread_unordered_store:
598; SKIP-CACHE-INV:       ; %bb.0: ; %entry
599; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
600; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
601; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
602; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
603; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
604; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
605; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
606; SKIP-CACHE-INV-NEXT:    s_endpgm
607;
608; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_unordered_store:
609; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
610; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
611; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
612; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
613; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
614; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
615; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
616; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
617;
618; GFX90A-TGSPLIT-LABEL: flat_singlethread_unordered_store:
619; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
620; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
621; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
622; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
623; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
624; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
625; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
626; GFX90A-TGSPLIT-NEXT:    s_endpgm
627;
628; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_unordered_store:
629; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
630; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
631; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
632; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
633; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
634; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
635; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
636; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
637;
638; GFX940-TGSPLIT-LABEL: flat_singlethread_unordered_store:
639; GFX940-TGSPLIT:       ; %bb.0: ; %entry
640; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
641; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
642; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
643; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
644; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
645; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
646; GFX940-TGSPLIT-NEXT:    s_endpgm
647;
648; GFX11-WGP-LABEL: flat_singlethread_unordered_store:
649; GFX11-WGP:       ; %bb.0: ; %entry
650; GFX11-WGP-NEXT:    s_clause 0x1
651; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
652; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
653; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
654; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
655; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
656; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
657; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
658; GFX11-WGP-NEXT:    s_endpgm
659;
660; GFX11-CU-LABEL: flat_singlethread_unordered_store:
661; GFX11-CU:       ; %bb.0: ; %entry
662; GFX11-CU-NEXT:    s_clause 0x1
663; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
664; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
665; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
666; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
667; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
668; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
669; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
670; GFX11-CU-NEXT:    s_endpgm
671    i32 %in, i32* %out) {
672entry:
673  store atomic i32 %in, i32* %out syncscope("singlethread") unordered, align 4
674  ret void
675}
676
677define amdgpu_kernel void @flat_singlethread_monotonic_store(
678; GFX7-LABEL: flat_singlethread_monotonic_store:
679; GFX7:       ; %bb.0: ; %entry
680; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
681; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
682; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
683; GFX7-NEXT:    v_mov_b32_e32 v0, s0
684; GFX7-NEXT:    v_mov_b32_e32 v1, s1
685; GFX7-NEXT:    v_mov_b32_e32 v2, s2
686; GFX7-NEXT:    flat_store_dword v[0:1], v2
687; GFX7-NEXT:    s_endpgm
688;
689; GFX10-WGP-LABEL: flat_singlethread_monotonic_store:
690; GFX10-WGP:       ; %bb.0: ; %entry
691; GFX10-WGP-NEXT:    s_clause 0x1
692; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
693; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
694; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
695; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
696; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
697; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
698; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
699; GFX10-WGP-NEXT:    s_endpgm
700;
701; GFX10-CU-LABEL: flat_singlethread_monotonic_store:
702; GFX10-CU:       ; %bb.0: ; %entry
703; GFX10-CU-NEXT:    s_clause 0x1
704; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
705; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
706; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
707; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
708; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
709; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
710; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
711; GFX10-CU-NEXT:    s_endpgm
712;
713; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_store:
714; SKIP-CACHE-INV:       ; %bb.0: ; %entry
715; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
716; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
717; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
718; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
719; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
720; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
721; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
722; SKIP-CACHE-INV-NEXT:    s_endpgm
723;
724; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_store:
725; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
726; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
727; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
728; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
729; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
730; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
731; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
732; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
733;
734; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_store:
735; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
736; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
737; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
738; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
739; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
740; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
741; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
742; GFX90A-TGSPLIT-NEXT:    s_endpgm
743;
744; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_store:
745; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
746; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
747; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
748; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
749; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
750; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
751; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
752; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
753;
754; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_store:
755; GFX940-TGSPLIT:       ; %bb.0: ; %entry
756; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
757; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
758; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
759; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
760; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
761; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
762; GFX940-TGSPLIT-NEXT:    s_endpgm
763;
764; GFX11-WGP-LABEL: flat_singlethread_monotonic_store:
765; GFX11-WGP:       ; %bb.0: ; %entry
766; GFX11-WGP-NEXT:    s_clause 0x1
767; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
768; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
769; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
770; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
771; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
772; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
773; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
774; GFX11-WGP-NEXT:    s_endpgm
775;
776; GFX11-CU-LABEL: flat_singlethread_monotonic_store:
777; GFX11-CU:       ; %bb.0: ; %entry
778; GFX11-CU-NEXT:    s_clause 0x1
779; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
780; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
781; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
782; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
783; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
784; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
785; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
786; GFX11-CU-NEXT:    s_endpgm
787    i32 %in, i32* %out) {
788entry:
789  store atomic i32 %in, i32* %out syncscope("singlethread") monotonic, align 4
790  ret void
791}
792
793define amdgpu_kernel void @flat_singlethread_release_store(
794; GFX7-LABEL: flat_singlethread_release_store:
795; GFX7:       ; %bb.0: ; %entry
796; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
797; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
798; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
799; GFX7-NEXT:    v_mov_b32_e32 v0, s0
800; GFX7-NEXT:    v_mov_b32_e32 v1, s1
801; GFX7-NEXT:    v_mov_b32_e32 v2, s2
802; GFX7-NEXT:    flat_store_dword v[0:1], v2
803; GFX7-NEXT:    s_endpgm
804;
805; GFX10-WGP-LABEL: flat_singlethread_release_store:
806; GFX10-WGP:       ; %bb.0: ; %entry
807; GFX10-WGP-NEXT:    s_clause 0x1
808; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
809; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
810; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
811; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
812; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
813; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
814; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
815; GFX10-WGP-NEXT:    s_endpgm
816;
817; GFX10-CU-LABEL: flat_singlethread_release_store:
818; GFX10-CU:       ; %bb.0: ; %entry
819; GFX10-CU-NEXT:    s_clause 0x1
820; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
821; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
822; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
823; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
824; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
825; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
826; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
827; GFX10-CU-NEXT:    s_endpgm
828;
829; SKIP-CACHE-INV-LABEL: flat_singlethread_release_store:
830; SKIP-CACHE-INV:       ; %bb.0: ; %entry
831; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
832; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
833; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
834; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
835; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
836; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
837; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
838; SKIP-CACHE-INV-NEXT:    s_endpgm
839;
840; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_store:
841; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
842; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
843; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
844; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
845; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
846; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
847; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
848; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
849;
850; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_store:
851; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
852; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
853; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
854; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
855; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
856; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
857; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
858; GFX90A-TGSPLIT-NEXT:    s_endpgm
859;
860; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_store:
861; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
862; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
863; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
864; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
865; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
866; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
867; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
868; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
869;
870; GFX940-TGSPLIT-LABEL: flat_singlethread_release_store:
871; GFX940-TGSPLIT:       ; %bb.0: ; %entry
872; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
873; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
874; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
875; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
876; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
877; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
878; GFX940-TGSPLIT-NEXT:    s_endpgm
879;
880; GFX11-WGP-LABEL: flat_singlethread_release_store:
881; GFX11-WGP:       ; %bb.0: ; %entry
882; GFX11-WGP-NEXT:    s_clause 0x1
883; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
884; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
885; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
886; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
887; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
888; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
889; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
890; GFX11-WGP-NEXT:    s_endpgm
891;
892; GFX11-CU-LABEL: flat_singlethread_release_store:
893; GFX11-CU:       ; %bb.0: ; %entry
894; GFX11-CU-NEXT:    s_clause 0x1
895; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
896; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
897; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
898; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
899; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
900; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
901; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
902; GFX11-CU-NEXT:    s_endpgm
903    i32 %in, i32* %out) {
904entry:
905  store atomic i32 %in, i32* %out syncscope("singlethread") release, align 4
906  ret void
907}
908
909define amdgpu_kernel void @flat_singlethread_seq_cst_store(
910; GFX7-LABEL: flat_singlethread_seq_cst_store:
911; GFX7:       ; %bb.0: ; %entry
912; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
913; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
914; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
915; GFX7-NEXT:    v_mov_b32_e32 v0, s0
916; GFX7-NEXT:    v_mov_b32_e32 v1, s1
917; GFX7-NEXT:    v_mov_b32_e32 v2, s2
918; GFX7-NEXT:    flat_store_dword v[0:1], v2
919; GFX7-NEXT:    s_endpgm
920;
921; GFX10-WGP-LABEL: flat_singlethread_seq_cst_store:
922; GFX10-WGP:       ; %bb.0: ; %entry
923; GFX10-WGP-NEXT:    s_clause 0x1
924; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
925; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
926; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
927; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
928; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
929; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
930; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
931; GFX10-WGP-NEXT:    s_endpgm
932;
933; GFX10-CU-LABEL: flat_singlethread_seq_cst_store:
934; GFX10-CU:       ; %bb.0: ; %entry
935; GFX10-CU-NEXT:    s_clause 0x1
936; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
937; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
938; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
939; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
940; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
941; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
942; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
943; GFX10-CU-NEXT:    s_endpgm
944;
945; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_store:
946; SKIP-CACHE-INV:       ; %bb.0: ; %entry
947; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
948; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
949; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
950; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
951; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
952; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
953; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
954; SKIP-CACHE-INV-NEXT:    s_endpgm
955;
956; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_store:
957; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
958; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
959; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
960; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
961; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
962; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
963; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
964; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
965;
966; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_store:
967; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
968; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
969; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
970; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
971; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
972; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
973; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
974; GFX90A-TGSPLIT-NEXT:    s_endpgm
975;
976; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_store:
977; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
978; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
979; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
980; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
981; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
982; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
983; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
984; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
985;
986; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_store:
987; GFX940-TGSPLIT:       ; %bb.0: ; %entry
988; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
989; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
990; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
991; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
992; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
993; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
994; GFX940-TGSPLIT-NEXT:    s_endpgm
995;
996; GFX11-WGP-LABEL: flat_singlethread_seq_cst_store:
997; GFX11-WGP:       ; %bb.0: ; %entry
998; GFX11-WGP-NEXT:    s_clause 0x1
999; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
1000; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
1001; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1002; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1003; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1004; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1005; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
1006; GFX11-WGP-NEXT:    s_endpgm
1007;
1008; GFX11-CU-LABEL: flat_singlethread_seq_cst_store:
1009; GFX11-CU:       ; %bb.0: ; %entry
1010; GFX11-CU-NEXT:    s_clause 0x1
1011; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
1012; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
1013; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1014; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1015; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1016; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1017; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1018; GFX11-CU-NEXT:    s_endpgm
1019    i32 %in, i32* %out) {
1020entry:
1021  store atomic i32 %in, i32* %out syncscope("singlethread") seq_cst, align 4
1022  ret void
1023}
1024
1025define amdgpu_kernel void @flat_singlethread_monotonic_atomicrmw(
1026; GFX7-LABEL: flat_singlethread_monotonic_atomicrmw:
1027; GFX7:       ; %bb.0: ; %entry
1028; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1029; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1030; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1031; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1032; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1033; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1034; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1035; GFX7-NEXT:    s_endpgm
1036;
1037; GFX10-WGP-LABEL: flat_singlethread_monotonic_atomicrmw:
1038; GFX10-WGP:       ; %bb.0: ; %entry
1039; GFX10-WGP-NEXT:    s_clause 0x1
1040; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1041; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1042; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1043; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1044; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1045; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1046; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1047; GFX10-WGP-NEXT:    s_endpgm
1048;
1049; GFX10-CU-LABEL: flat_singlethread_monotonic_atomicrmw:
1050; GFX10-CU:       ; %bb.0: ; %entry
1051; GFX10-CU-NEXT:    s_clause 0x1
1052; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1053; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1054; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1055; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1056; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1057; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1058; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1059; GFX10-CU-NEXT:    s_endpgm
1060;
1061; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_atomicrmw:
1062; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1063; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1064; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1065; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1066; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1067; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1068; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1069; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1070; SKIP-CACHE-INV-NEXT:    s_endpgm
1071;
1072; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_atomicrmw:
1073; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1074; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1075; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1076; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1077; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1078; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1079; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1080; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1081;
1082; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_atomicrmw:
1083; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1084; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1085; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1086; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1087; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1088; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1089; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1090; GFX90A-TGSPLIT-NEXT:    s_endpgm
1091;
1092; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_atomicrmw:
1093; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1094; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1095; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1096; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1097; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1098; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1099; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1100; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1101;
1102; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_atomicrmw:
1103; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1104; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1105; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1106; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1107; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1108; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1109; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1110; GFX940-TGSPLIT-NEXT:    s_endpgm
1111;
1112; GFX11-WGP-LABEL: flat_singlethread_monotonic_atomicrmw:
1113; GFX11-WGP:       ; %bb.0: ; %entry
1114; GFX11-WGP-NEXT:    s_clause 0x1
1115; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1116; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1117; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1118; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1119; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1120; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1121; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1122; GFX11-WGP-NEXT:    s_endpgm
1123;
1124; GFX11-CU-LABEL: flat_singlethread_monotonic_atomicrmw:
1125; GFX11-CU:       ; %bb.0: ; %entry
1126; GFX11-CU-NEXT:    s_clause 0x1
1127; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1128; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1129; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1130; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1131; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1132; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1133; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1134; GFX11-CU-NEXT:    s_endpgm
1135    i32* %out, i32 %in) {
1136entry:
1137  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") monotonic
1138  ret void
1139}
1140
1141define amdgpu_kernel void @flat_singlethread_acquire_atomicrmw(
1142; GFX7-LABEL: flat_singlethread_acquire_atomicrmw:
1143; GFX7:       ; %bb.0: ; %entry
1144; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1145; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1146; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1147; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1148; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1149; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1150; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1151; GFX7-NEXT:    s_endpgm
1152;
1153; GFX10-WGP-LABEL: flat_singlethread_acquire_atomicrmw:
1154; GFX10-WGP:       ; %bb.0: ; %entry
1155; GFX10-WGP-NEXT:    s_clause 0x1
1156; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1157; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1158; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1159; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1160; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1161; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1162; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1163; GFX10-WGP-NEXT:    s_endpgm
1164;
1165; GFX10-CU-LABEL: flat_singlethread_acquire_atomicrmw:
1166; GFX10-CU:       ; %bb.0: ; %entry
1167; GFX10-CU-NEXT:    s_clause 0x1
1168; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1169; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1170; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1171; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1172; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1173; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1174; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1175; GFX10-CU-NEXT:    s_endpgm
1176;
1177; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_atomicrmw:
1178; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1179; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1180; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1181; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1182; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1183; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1184; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1185; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1186; SKIP-CACHE-INV-NEXT:    s_endpgm
1187;
1188; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_atomicrmw:
1189; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1190; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1191; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1192; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1193; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1194; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1195; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1196; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1197;
1198; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_atomicrmw:
1199; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1200; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1201; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1202; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1203; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1204; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1205; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1206; GFX90A-TGSPLIT-NEXT:    s_endpgm
1207;
1208; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_atomicrmw:
1209; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1210; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1211; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1212; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1213; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1214; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1215; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1216; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1217;
1218; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_atomicrmw:
1219; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1220; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1221; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1222; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1223; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1224; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1225; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1226; GFX940-TGSPLIT-NEXT:    s_endpgm
1227;
1228; GFX11-WGP-LABEL: flat_singlethread_acquire_atomicrmw:
1229; GFX11-WGP:       ; %bb.0: ; %entry
1230; GFX11-WGP-NEXT:    s_clause 0x1
1231; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1232; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1233; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1234; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1235; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1236; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1237; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1238; GFX11-WGP-NEXT:    s_endpgm
1239;
1240; GFX11-CU-LABEL: flat_singlethread_acquire_atomicrmw:
1241; GFX11-CU:       ; %bb.0: ; %entry
1242; GFX11-CU-NEXT:    s_clause 0x1
1243; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1244; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1245; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1246; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1247; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1248; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1249; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1250; GFX11-CU-NEXT:    s_endpgm
1251    i32* %out, i32 %in) {
1252entry:
1253  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acquire
1254  ret void
1255}
1256
1257define amdgpu_kernel void @flat_singlethread_release_atomicrmw(
1258; GFX7-LABEL: flat_singlethread_release_atomicrmw:
1259; GFX7:       ; %bb.0: ; %entry
1260; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1261; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1262; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1263; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1264; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1265; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1266; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1267; GFX7-NEXT:    s_endpgm
1268;
1269; GFX10-WGP-LABEL: flat_singlethread_release_atomicrmw:
1270; GFX10-WGP:       ; %bb.0: ; %entry
1271; GFX10-WGP-NEXT:    s_clause 0x1
1272; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1273; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1274; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1275; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1276; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1277; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1278; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1279; GFX10-WGP-NEXT:    s_endpgm
1280;
1281; GFX10-CU-LABEL: flat_singlethread_release_atomicrmw:
1282; GFX10-CU:       ; %bb.0: ; %entry
1283; GFX10-CU-NEXT:    s_clause 0x1
1284; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1285; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1286; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1287; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1288; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1289; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1290; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1291; GFX10-CU-NEXT:    s_endpgm
1292;
1293; SKIP-CACHE-INV-LABEL: flat_singlethread_release_atomicrmw:
1294; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1295; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1296; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1297; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1298; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1299; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1300; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1301; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1302; SKIP-CACHE-INV-NEXT:    s_endpgm
1303;
1304; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_atomicrmw:
1305; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1306; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1307; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1308; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1309; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1310; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1311; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1312; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1313;
1314; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_atomicrmw:
1315; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1316; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1317; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1318; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1319; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1320; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1321; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1322; GFX90A-TGSPLIT-NEXT:    s_endpgm
1323;
1324; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_atomicrmw:
1325; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1326; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1327; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1328; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1329; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1330; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1331; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1332; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1333;
1334; GFX940-TGSPLIT-LABEL: flat_singlethread_release_atomicrmw:
1335; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1336; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1337; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1338; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1339; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1340; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1341; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1342; GFX940-TGSPLIT-NEXT:    s_endpgm
1343;
1344; GFX11-WGP-LABEL: flat_singlethread_release_atomicrmw:
1345; GFX11-WGP:       ; %bb.0: ; %entry
1346; GFX11-WGP-NEXT:    s_clause 0x1
1347; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1348; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1349; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1350; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1351; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1352; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1353; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1354; GFX11-WGP-NEXT:    s_endpgm
1355;
1356; GFX11-CU-LABEL: flat_singlethread_release_atomicrmw:
1357; GFX11-CU:       ; %bb.0: ; %entry
1358; GFX11-CU-NEXT:    s_clause 0x1
1359; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1360; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1361; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1362; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1363; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1364; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1365; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1366; GFX11-CU-NEXT:    s_endpgm
1367    i32* %out, i32 %in) {
1368entry:
1369  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") release
1370  ret void
1371}
1372
1373define amdgpu_kernel void @flat_singlethread_acq_rel_atomicrmw(
1374; GFX7-LABEL: flat_singlethread_acq_rel_atomicrmw:
1375; GFX7:       ; %bb.0: ; %entry
1376; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1377; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1378; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1379; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1380; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1381; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1382; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1383; GFX7-NEXT:    s_endpgm
1384;
1385; GFX10-WGP-LABEL: flat_singlethread_acq_rel_atomicrmw:
1386; GFX10-WGP:       ; %bb.0: ; %entry
1387; GFX10-WGP-NEXT:    s_clause 0x1
1388; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1389; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1390; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1391; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1392; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1393; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1394; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1395; GFX10-WGP-NEXT:    s_endpgm
1396;
1397; GFX10-CU-LABEL: flat_singlethread_acq_rel_atomicrmw:
1398; GFX10-CU:       ; %bb.0: ; %entry
1399; GFX10-CU-NEXT:    s_clause 0x1
1400; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1401; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1402; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1403; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1404; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1405; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1406; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1407; GFX10-CU-NEXT:    s_endpgm
1408;
1409; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_atomicrmw:
1410; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1411; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1412; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1413; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1414; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1415; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1416; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1417; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1418; SKIP-CACHE-INV-NEXT:    s_endpgm
1419;
1420; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_atomicrmw:
1421; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1422; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1423; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1424; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1425; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1426; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1427; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1428; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1429;
1430; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_atomicrmw:
1431; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1432; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1433; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1434; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1435; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1436; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1437; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1438; GFX90A-TGSPLIT-NEXT:    s_endpgm
1439;
1440; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_atomicrmw:
1441; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1442; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1443; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1444; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1445; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1446; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1447; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1448; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1449;
1450; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_atomicrmw:
1451; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1452; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1453; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1454; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1455; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1456; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1457; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1458; GFX940-TGSPLIT-NEXT:    s_endpgm
1459;
1460; GFX11-WGP-LABEL: flat_singlethread_acq_rel_atomicrmw:
1461; GFX11-WGP:       ; %bb.0: ; %entry
1462; GFX11-WGP-NEXT:    s_clause 0x1
1463; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1464; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1465; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1466; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1467; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1468; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1469; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1470; GFX11-WGP-NEXT:    s_endpgm
1471;
1472; GFX11-CU-LABEL: flat_singlethread_acq_rel_atomicrmw:
1473; GFX11-CU:       ; %bb.0: ; %entry
1474; GFX11-CU-NEXT:    s_clause 0x1
1475; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1476; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1477; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1478; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1479; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1480; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1481; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1482; GFX11-CU-NEXT:    s_endpgm
1483    i32* %out, i32 %in) {
1484entry:
1485  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acq_rel
1486  ret void
1487}
1488
1489define amdgpu_kernel void @flat_singlethread_seq_cst_atomicrmw(
1490; GFX7-LABEL: flat_singlethread_seq_cst_atomicrmw:
1491; GFX7:       ; %bb.0: ; %entry
1492; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1493; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1494; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1495; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1496; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1497; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1498; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1499; GFX7-NEXT:    s_endpgm
1500;
1501; GFX10-WGP-LABEL: flat_singlethread_seq_cst_atomicrmw:
1502; GFX10-WGP:       ; %bb.0: ; %entry
1503; GFX10-WGP-NEXT:    s_clause 0x1
1504; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1505; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1506; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1507; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1508; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1509; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1510; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1511; GFX10-WGP-NEXT:    s_endpgm
1512;
1513; GFX10-CU-LABEL: flat_singlethread_seq_cst_atomicrmw:
1514; GFX10-CU:       ; %bb.0: ; %entry
1515; GFX10-CU-NEXT:    s_clause 0x1
1516; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1517; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1518; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1519; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1520; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1521; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1522; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1523; GFX10-CU-NEXT:    s_endpgm
1524;
1525; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_atomicrmw:
1526; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1527; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1528; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1529; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1530; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1531; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1532; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1533; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1534; SKIP-CACHE-INV-NEXT:    s_endpgm
1535;
1536; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_atomicrmw:
1537; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1538; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1539; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1540; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1541; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1542; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1543; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1544; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1545;
1546; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_atomicrmw:
1547; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1548; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1549; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1550; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1551; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1552; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1553; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1554; GFX90A-TGSPLIT-NEXT:    s_endpgm
1555;
1556; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_atomicrmw:
1557; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1558; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1559; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1560; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1561; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1562; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1563; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1564; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1565;
1566; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_atomicrmw:
1567; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1568; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1569; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1570; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1571; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1572; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1573; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1574; GFX940-TGSPLIT-NEXT:    s_endpgm
1575;
1576; GFX11-WGP-LABEL: flat_singlethread_seq_cst_atomicrmw:
1577; GFX11-WGP:       ; %bb.0: ; %entry
1578; GFX11-WGP-NEXT:    s_clause 0x1
1579; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1580; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1581; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1582; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1583; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1584; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1585; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1586; GFX11-WGP-NEXT:    s_endpgm
1587;
1588; GFX11-CU-LABEL: flat_singlethread_seq_cst_atomicrmw:
1589; GFX11-CU:       ; %bb.0: ; %entry
1590; GFX11-CU-NEXT:    s_clause 0x1
1591; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1592; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1593; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1594; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1595; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1596; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1597; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1598; GFX11-CU-NEXT:    s_endpgm
1599    i32* %out, i32 %in) {
1600entry:
1601  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") seq_cst
1602  ret void
1603}
1604
1605define amdgpu_kernel void @flat_singlethread_acquire_ret_atomicrmw(
1606; GFX7-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1607; GFX7:       ; %bb.0: ; %entry
1608; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1609; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1610; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1611; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1612; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1613; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1614; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1615; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1616; GFX7-NEXT:    flat_store_dword v[0:1], v2
1617; GFX7-NEXT:    s_endpgm
1618;
1619; GFX10-WGP-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1620; GFX10-WGP:       ; %bb.0: ; %entry
1621; GFX10-WGP-NEXT:    s_clause 0x1
1622; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1623; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1624; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1625; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1626; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1627; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1628; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1629; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1630; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1631; GFX10-WGP-NEXT:    s_endpgm
1632;
1633; GFX10-CU-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1634; GFX10-CU:       ; %bb.0: ; %entry
1635; GFX10-CU-NEXT:    s_clause 0x1
1636; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1637; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1638; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1639; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1640; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1641; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1642; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1643; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1644; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1645; GFX10-CU-NEXT:    s_endpgm
1646;
1647; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1648; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1649; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1650; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1651; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1652; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1653; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1654; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1655; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1656; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1657; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1658; SKIP-CACHE-INV-NEXT:    s_endpgm
1659;
1660; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1661; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1662; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1663; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1664; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1665; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1666; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1667; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1668; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1669; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1670; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1671;
1672; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1673; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1674; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1675; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1676; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1677; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1678; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1679; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1680; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1681; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1682; GFX90A-TGSPLIT-NEXT:    s_endpgm
1683;
1684; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1685; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1686; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1687; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1688; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1689; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1690; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1691; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1692; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1693; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1694; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1695;
1696; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1697; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1698; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1699; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1700; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1701; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1702; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1703; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1704; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1705; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1706; GFX940-TGSPLIT-NEXT:    s_endpgm
1707;
1708; GFX11-WGP-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1709; GFX11-WGP:       ; %bb.0: ; %entry
1710; GFX11-WGP-NEXT:    s_clause 0x1
1711; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1712; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1713; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1714; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1715; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1716; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1717; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1718; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1719; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
1720; GFX11-WGP-NEXT:    s_endpgm
1721;
1722; GFX11-CU-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1723; GFX11-CU:       ; %bb.0: ; %entry
1724; GFX11-CU-NEXT:    s_clause 0x1
1725; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1726; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1727; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1728; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1729; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1730; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1731; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1732; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1733; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1734; GFX11-CU-NEXT:    s_endpgm
1735    i32* %out, i32 %in) {
1736entry:
1737  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acquire
1738  store i32 %val, i32* %out, align 4
1739  ret void
1740}
1741
1742define amdgpu_kernel void @flat_singlethread_acq_rel_ret_atomicrmw(
1743; GFX7-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1744; GFX7:       ; %bb.0: ; %entry
1745; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1746; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1747; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1748; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1749; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1750; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1751; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1752; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1753; GFX7-NEXT:    flat_store_dword v[0:1], v2
1754; GFX7-NEXT:    s_endpgm
1755;
1756; GFX10-WGP-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1757; GFX10-WGP:       ; %bb.0: ; %entry
1758; GFX10-WGP-NEXT:    s_clause 0x1
1759; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1760; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1761; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1762; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1763; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1764; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1765; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1766; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1767; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1768; GFX10-WGP-NEXT:    s_endpgm
1769;
1770; GFX10-CU-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1771; GFX10-CU:       ; %bb.0: ; %entry
1772; GFX10-CU-NEXT:    s_clause 0x1
1773; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1774; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1775; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1776; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1777; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1778; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1779; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1780; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1781; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1782; GFX10-CU-NEXT:    s_endpgm
1783;
1784; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1785; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1786; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1787; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1788; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1789; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1790; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1791; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1792; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1793; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1794; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1795; SKIP-CACHE-INV-NEXT:    s_endpgm
1796;
1797; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1798; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1799; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1800; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1801; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1802; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1803; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1804; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1805; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1806; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1807; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1808;
1809; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1810; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1811; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1812; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1813; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1814; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1815; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1816; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1817; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1818; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1819; GFX90A-TGSPLIT-NEXT:    s_endpgm
1820;
1821; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1822; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1823; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1824; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1825; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1826; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1827; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1828; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1829; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1830; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1831; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1832;
1833; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1834; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1835; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1836; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1837; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1838; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1839; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1840; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1841; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1842; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1843; GFX940-TGSPLIT-NEXT:    s_endpgm
1844;
1845; GFX11-WGP-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1846; GFX11-WGP:       ; %bb.0: ; %entry
1847; GFX11-WGP-NEXT:    s_clause 0x1
1848; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1849; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1850; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1851; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1852; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1853; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1854; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1855; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1856; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
1857; GFX11-WGP-NEXT:    s_endpgm
1858;
1859; GFX11-CU-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1860; GFX11-CU:       ; %bb.0: ; %entry
1861; GFX11-CU-NEXT:    s_clause 0x1
1862; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1863; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1864; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1865; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1866; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1867; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1868; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1869; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1870; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1871; GFX11-CU-NEXT:    s_endpgm
1872    i32* %out, i32 %in) {
1873entry:
1874  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acq_rel
1875  store i32 %val, i32* %out, align 4
1876  ret void
1877}
1878
1879define amdgpu_kernel void @flat_singlethread_seq_cst_ret_atomicrmw(
1880; GFX7-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1881; GFX7:       ; %bb.0: ; %entry
1882; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1883; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1884; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1885; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1886; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1887; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1888; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1889; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1890; GFX7-NEXT:    flat_store_dword v[0:1], v2
1891; GFX7-NEXT:    s_endpgm
1892;
1893; GFX10-WGP-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1894; GFX10-WGP:       ; %bb.0: ; %entry
1895; GFX10-WGP-NEXT:    s_clause 0x1
1896; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1897; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1898; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1899; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1900; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1901; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1902; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1903; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1904; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1905; GFX10-WGP-NEXT:    s_endpgm
1906;
1907; GFX10-CU-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1908; GFX10-CU:       ; %bb.0: ; %entry
1909; GFX10-CU-NEXT:    s_clause 0x1
1910; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1911; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1912; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1913; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1914; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1915; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1916; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1917; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1918; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1919; GFX10-CU-NEXT:    s_endpgm
1920;
1921; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1922; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1923; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1924; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1925; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1926; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1927; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1928; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1929; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1930; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1931; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1932; SKIP-CACHE-INV-NEXT:    s_endpgm
1933;
1934; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1935; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1936; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1937; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1938; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1939; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1940; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1941; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1942; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1943; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1944; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1945;
1946; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1947; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1948; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1949; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1950; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1951; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1952; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1953; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1954; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1955; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1956; GFX90A-TGSPLIT-NEXT:    s_endpgm
1957;
1958; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1959; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1960; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1961; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1962; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1963; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1964; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1965; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1966; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1967; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1968; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1969;
1970; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1971; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1972; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1973; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1974; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1975; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1976; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1977; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1978; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1979; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1980; GFX940-TGSPLIT-NEXT:    s_endpgm
1981;
1982; GFX11-WGP-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1983; GFX11-WGP:       ; %bb.0: ; %entry
1984; GFX11-WGP-NEXT:    s_clause 0x1
1985; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1986; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1987; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1988; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1989; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1990; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1991; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1992; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1993; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
1994; GFX11-WGP-NEXT:    s_endpgm
1995;
1996; GFX11-CU-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1997; GFX11-CU:       ; %bb.0: ; %entry
1998; GFX11-CU-NEXT:    s_clause 0x1
1999; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2000; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
2001; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2002; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2003; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2004; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2005; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2006; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2007; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
2008; GFX11-CU-NEXT:    s_endpgm
2009    i32* %out, i32 %in) {
2010entry:
2011  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") seq_cst
2012  store i32 %val, i32* %out, align 4
2013  ret void
2014}
2015
2016define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_cmpxchg(
2017; GFX7-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
2018; GFX7:       ; %bb.0: ; %entry
2019; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2020; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2021; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2022; GFX7-NEXT:    s_add_u32 s0, s0, 16
2023; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2024; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2025; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2026; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2027; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2028; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2029; GFX7-NEXT:    s_endpgm
2030;
2031; GFX10-WGP-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
2032; GFX10-WGP:       ; %bb.0: ; %entry
2033; GFX10-WGP-NEXT:    s_clause 0x1
2034; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2035; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2036; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2037; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2038; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2039; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2040; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2041; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2042; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2043; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2044; GFX10-WGP-NEXT:    s_endpgm
2045;
2046; GFX10-CU-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
2047; GFX10-CU:       ; %bb.0: ; %entry
2048; GFX10-CU-NEXT:    s_clause 0x1
2049; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2050; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2051; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2052; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2053; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2054; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2055; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2056; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2057; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2058; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2059; GFX10-CU-NEXT:    s_endpgm
2060;
2061; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
2062; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2063; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2064; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2065; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2066; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2067; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2068; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2069; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2070; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2071; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2072; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2073; SKIP-CACHE-INV-NEXT:    s_endpgm
2074;
2075; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
2076; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2077; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2078; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2079; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2080; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2081; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2082; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2083; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2084;
2085; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
2086; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2087; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2088; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2089; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2090; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2091; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2092; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2093; GFX90A-TGSPLIT-NEXT:    s_endpgm
2094;
2095; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
2096; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2097; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2098; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2099; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2100; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2101; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2102; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2103; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2104;
2105; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
2106; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2107; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2108; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2109; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2110; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2111; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2112; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2113; GFX940-TGSPLIT-NEXT:    s_endpgm
2114;
2115; GFX11-WGP-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
2116; GFX11-WGP:       ; %bb.0: ; %entry
2117; GFX11-WGP-NEXT:    s_clause 0x1
2118; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2119; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2120; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2121; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2122; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2123; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2124; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2125; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2126; GFX11-WGP-NEXT:    s_endpgm
2127;
2128; GFX11-CU-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
2129; GFX11-CU:       ; %bb.0: ; %entry
2130; GFX11-CU-NEXT:    s_clause 0x1
2131; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2132; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2133; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2134; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2135; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2136; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2137; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2138; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2139; GFX11-CU-NEXT:    s_endpgm
2140    i32* %out, i32 %in, i32 %old) {
2141entry:
2142  %gep = getelementptr i32, i32* %out, i32 4
2143  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
2144  ret void
2145}
2146
2147define amdgpu_kernel void @flat_singlethread_acquire_monotonic_cmpxchg(
2148; GFX7-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
2149; GFX7:       ; %bb.0: ; %entry
2150; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2151; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2152; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2153; GFX7-NEXT:    s_add_u32 s0, s0, 16
2154; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2155; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2156; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2157; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2158; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2159; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2160; GFX7-NEXT:    s_endpgm
2161;
2162; GFX10-WGP-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
2163; GFX10-WGP:       ; %bb.0: ; %entry
2164; GFX10-WGP-NEXT:    s_clause 0x1
2165; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2166; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2167; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2168; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2169; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2170; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2171; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2172; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2173; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2174; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2175; GFX10-WGP-NEXT:    s_endpgm
2176;
2177; GFX10-CU-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
2178; GFX10-CU:       ; %bb.0: ; %entry
2179; GFX10-CU-NEXT:    s_clause 0x1
2180; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2181; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2182; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2183; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2184; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2185; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2186; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2187; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2188; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2189; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2190; GFX10-CU-NEXT:    s_endpgm
2191;
2192; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
2193; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2194; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2195; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2196; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2197; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2198; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2199; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2200; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2201; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2202; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2203; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2204; SKIP-CACHE-INV-NEXT:    s_endpgm
2205;
2206; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
2207; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2208; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2209; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2210; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2211; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2212; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2213; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2214; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2215;
2216; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
2217; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2218; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2219; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2220; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2221; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2222; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2223; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2224; GFX90A-TGSPLIT-NEXT:    s_endpgm
2225;
2226; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
2227; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2228; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2229; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2230; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2231; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2232; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2233; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2234; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2235;
2236; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
2237; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2238; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2239; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2240; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2241; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2242; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2243; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2244; GFX940-TGSPLIT-NEXT:    s_endpgm
2245;
2246; GFX11-WGP-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
2247; GFX11-WGP:       ; %bb.0: ; %entry
2248; GFX11-WGP-NEXT:    s_clause 0x1
2249; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2250; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2251; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2252; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2253; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2254; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2255; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2256; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2257; GFX11-WGP-NEXT:    s_endpgm
2258;
2259; GFX11-CU-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
2260; GFX11-CU:       ; %bb.0: ; %entry
2261; GFX11-CU-NEXT:    s_clause 0x1
2262; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2263; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2264; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2265; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2266; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2267; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2268; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2269; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2270; GFX11-CU-NEXT:    s_endpgm
2271    i32* %out, i32 %in, i32 %old) {
2272entry:
2273  %gep = getelementptr i32, i32* %out, i32 4
2274  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
2275  ret void
2276}
2277
2278define amdgpu_kernel void @flat_singlethread_release_monotonic_cmpxchg(
2279; GFX7-LABEL: flat_singlethread_release_monotonic_cmpxchg:
2280; GFX7:       ; %bb.0: ; %entry
2281; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2282; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2283; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2284; GFX7-NEXT:    s_add_u32 s0, s0, 16
2285; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2286; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2287; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2288; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2289; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2290; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2291; GFX7-NEXT:    s_endpgm
2292;
2293; GFX10-WGP-LABEL: flat_singlethread_release_monotonic_cmpxchg:
2294; GFX10-WGP:       ; %bb.0: ; %entry
2295; GFX10-WGP-NEXT:    s_clause 0x1
2296; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2297; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2298; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2299; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2300; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2301; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2302; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2303; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2304; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2305; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2306; GFX10-WGP-NEXT:    s_endpgm
2307;
2308; GFX10-CU-LABEL: flat_singlethread_release_monotonic_cmpxchg:
2309; GFX10-CU:       ; %bb.0: ; %entry
2310; GFX10-CU-NEXT:    s_clause 0x1
2311; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2312; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2313; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2314; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2315; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2316; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2317; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2318; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2319; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2320; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2321; GFX10-CU-NEXT:    s_endpgm
2322;
2323; SKIP-CACHE-INV-LABEL: flat_singlethread_release_monotonic_cmpxchg:
2324; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2325; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2326; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2327; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2328; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2329; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2330; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2331; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2332; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2333; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2334; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2335; SKIP-CACHE-INV-NEXT:    s_endpgm
2336;
2337; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_monotonic_cmpxchg:
2338; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2339; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2340; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2341; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2342; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2343; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2344; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2345; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2346;
2347; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_monotonic_cmpxchg:
2348; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2349; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2350; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2351; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2352; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2353; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2354; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2355; GFX90A-TGSPLIT-NEXT:    s_endpgm
2356;
2357; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_monotonic_cmpxchg:
2358; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2359; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2360; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2361; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2362; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2363; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2364; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2365; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2366;
2367; GFX940-TGSPLIT-LABEL: flat_singlethread_release_monotonic_cmpxchg:
2368; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2369; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2370; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2371; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2372; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2373; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2374; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2375; GFX940-TGSPLIT-NEXT:    s_endpgm
2376;
2377; GFX11-WGP-LABEL: flat_singlethread_release_monotonic_cmpxchg:
2378; GFX11-WGP:       ; %bb.0: ; %entry
2379; GFX11-WGP-NEXT:    s_clause 0x1
2380; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2381; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2382; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2383; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2384; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2385; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2386; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2387; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2388; GFX11-WGP-NEXT:    s_endpgm
2389;
2390; GFX11-CU-LABEL: flat_singlethread_release_monotonic_cmpxchg:
2391; GFX11-CU:       ; %bb.0: ; %entry
2392; GFX11-CU-NEXT:    s_clause 0x1
2393; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2394; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2395; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2396; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2397; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2398; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2399; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2400; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2401; GFX11-CU-NEXT:    s_endpgm
2402    i32* %out, i32 %in, i32 %old) {
2403entry:
2404  %gep = getelementptr i32, i32* %out, i32 4
2405  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
2406  ret void
2407}
2408
2409define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_cmpxchg(
2410; GFX7-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2411; GFX7:       ; %bb.0: ; %entry
2412; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2413; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2414; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2415; GFX7-NEXT:    s_add_u32 s0, s0, 16
2416; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2417; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2418; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2419; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2420; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2421; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2422; GFX7-NEXT:    s_endpgm
2423;
2424; GFX10-WGP-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2425; GFX10-WGP:       ; %bb.0: ; %entry
2426; GFX10-WGP-NEXT:    s_clause 0x1
2427; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2428; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2429; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2430; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2431; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2432; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2433; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2434; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2435; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2436; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2437; GFX10-WGP-NEXT:    s_endpgm
2438;
2439; GFX10-CU-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2440; GFX10-CU:       ; %bb.0: ; %entry
2441; GFX10-CU-NEXT:    s_clause 0x1
2442; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2443; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2444; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2445; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2446; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2447; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2448; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2449; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2450; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2451; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2452; GFX10-CU-NEXT:    s_endpgm
2453;
2454; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2455; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2456; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2457; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2458; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2459; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2460; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2461; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2462; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2463; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2464; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2465; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2466; SKIP-CACHE-INV-NEXT:    s_endpgm
2467;
2468; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2469; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2470; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2471; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2472; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2473; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2474; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2475; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2476; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2477;
2478; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2479; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2480; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2481; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2482; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2483; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2484; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2485; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2486; GFX90A-TGSPLIT-NEXT:    s_endpgm
2487;
2488; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2489; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2490; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2491; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2492; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2493; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2494; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2495; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2496; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2497;
2498; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2499; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2500; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2501; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2502; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2503; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2504; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2505; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2506; GFX940-TGSPLIT-NEXT:    s_endpgm
2507;
2508; GFX11-WGP-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2509; GFX11-WGP:       ; %bb.0: ; %entry
2510; GFX11-WGP-NEXT:    s_clause 0x1
2511; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2512; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2513; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2514; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2515; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2516; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2517; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2518; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2519; GFX11-WGP-NEXT:    s_endpgm
2520;
2521; GFX11-CU-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2522; GFX11-CU:       ; %bb.0: ; %entry
2523; GFX11-CU-NEXT:    s_clause 0x1
2524; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2525; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2526; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2527; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2528; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2529; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2530; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2531; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2532; GFX11-CU-NEXT:    s_endpgm
2533    i32* %out, i32 %in, i32 %old) {
2534entry:
2535  %gep = getelementptr i32, i32* %out, i32 4
2536  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
2537  ret void
2538}
2539
2540define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_cmpxchg(
2541; GFX7-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2542; GFX7:       ; %bb.0: ; %entry
2543; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2544; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2545; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2546; GFX7-NEXT:    s_add_u32 s0, s0, 16
2547; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2548; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2549; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2550; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2551; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2552; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2553; GFX7-NEXT:    s_endpgm
2554;
2555; GFX10-WGP-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2556; GFX10-WGP:       ; %bb.0: ; %entry
2557; GFX10-WGP-NEXT:    s_clause 0x1
2558; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2559; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2560; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2561; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2562; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2563; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2564; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2565; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2566; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2567; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2568; GFX10-WGP-NEXT:    s_endpgm
2569;
2570; GFX10-CU-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2571; GFX10-CU:       ; %bb.0: ; %entry
2572; GFX10-CU-NEXT:    s_clause 0x1
2573; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2574; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2575; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2576; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2577; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2578; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2579; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2580; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2581; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2582; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2583; GFX10-CU-NEXT:    s_endpgm
2584;
2585; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2586; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2587; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2588; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2589; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2590; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2591; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2592; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2593; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2594; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2595; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2596; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2597; SKIP-CACHE-INV-NEXT:    s_endpgm
2598;
2599; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2600; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2601; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2602; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2603; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2604; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2605; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2606; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2607; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2608;
2609; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2610; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2611; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2612; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2613; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2614; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2615; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2616; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2617; GFX90A-TGSPLIT-NEXT:    s_endpgm
2618;
2619; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2620; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2621; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2622; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2623; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2624; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2625; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2626; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2627; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2628;
2629; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2630; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2631; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2632; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2633; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2634; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2635; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2636; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2637; GFX940-TGSPLIT-NEXT:    s_endpgm
2638;
2639; GFX11-WGP-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2640; GFX11-WGP:       ; %bb.0: ; %entry
2641; GFX11-WGP-NEXT:    s_clause 0x1
2642; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2643; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2644; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2645; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2646; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2647; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2648; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2649; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2650; GFX11-WGP-NEXT:    s_endpgm
2651;
2652; GFX11-CU-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2653; GFX11-CU:       ; %bb.0: ; %entry
2654; GFX11-CU-NEXT:    s_clause 0x1
2655; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2656; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2657; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2658; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2659; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2660; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2661; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2662; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2663; GFX11-CU-NEXT:    s_endpgm
2664    i32* %out, i32 %in, i32 %old) {
2665entry:
2666  %gep = getelementptr i32, i32* %out, i32 4
2667  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
2668  ret void
2669}
2670
2671define amdgpu_kernel void @flat_singlethread_monotonic_acquire_cmpxchg(
2672; GFX7-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2673; GFX7:       ; %bb.0: ; %entry
2674; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2675; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2676; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2677; GFX7-NEXT:    s_add_u32 s0, s0, 16
2678; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2679; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2680; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2681; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2682; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2683; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2684; GFX7-NEXT:    s_endpgm
2685;
2686; GFX10-WGP-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2687; GFX10-WGP:       ; %bb.0: ; %entry
2688; GFX10-WGP-NEXT:    s_clause 0x1
2689; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2690; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2691; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2692; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2693; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2694; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2695; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2696; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2697; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2698; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2699; GFX10-WGP-NEXT:    s_endpgm
2700;
2701; GFX10-CU-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2702; GFX10-CU:       ; %bb.0: ; %entry
2703; GFX10-CU-NEXT:    s_clause 0x1
2704; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2705; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2706; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2707; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2708; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2709; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2710; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2711; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2712; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2713; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2714; GFX10-CU-NEXT:    s_endpgm
2715;
2716; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2717; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2718; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2719; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2720; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2721; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2722; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2723; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2724; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2725; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2726; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2727; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2728; SKIP-CACHE-INV-NEXT:    s_endpgm
2729;
2730; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2731; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2732; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2733; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2734; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2735; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2736; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2737; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2738; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2739;
2740; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2741; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2742; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2743; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2744; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2745; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2746; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2747; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2748; GFX90A-TGSPLIT-NEXT:    s_endpgm
2749;
2750; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2751; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2752; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2753; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2754; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2755; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2756; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2757; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2758; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2759;
2760; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2761; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2762; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2763; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2764; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2765; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2766; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2767; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2768; GFX940-TGSPLIT-NEXT:    s_endpgm
2769;
2770; GFX11-WGP-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2771; GFX11-WGP:       ; %bb.0: ; %entry
2772; GFX11-WGP-NEXT:    s_clause 0x1
2773; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2774; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2775; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2776; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2777; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2778; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2779; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2780; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2781; GFX11-WGP-NEXT:    s_endpgm
2782;
2783; GFX11-CU-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2784; GFX11-CU:       ; %bb.0: ; %entry
2785; GFX11-CU-NEXT:    s_clause 0x1
2786; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2787; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2788; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2789; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2790; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2791; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2792; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2793; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2794; GFX11-CU-NEXT:    s_endpgm
2795    i32* %out, i32 %in, i32 %old) {
2796entry:
2797  %gep = getelementptr i32, i32* %out, i32 4
2798  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic acquire
2799  ret void
2800}
2801
2802define amdgpu_kernel void @flat_singlethread_acquire_acquire_cmpxchg(
2803; GFX7-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2804; GFX7:       ; %bb.0: ; %entry
2805; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2806; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2807; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2808; GFX7-NEXT:    s_add_u32 s0, s0, 16
2809; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2810; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2811; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2812; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2813; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2814; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2815; GFX7-NEXT:    s_endpgm
2816;
2817; GFX10-WGP-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2818; GFX10-WGP:       ; %bb.0: ; %entry
2819; GFX10-WGP-NEXT:    s_clause 0x1
2820; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2821; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2822; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2823; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2824; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2825; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2826; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2827; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2828; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2829; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2830; GFX10-WGP-NEXT:    s_endpgm
2831;
2832; GFX10-CU-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2833; GFX10-CU:       ; %bb.0: ; %entry
2834; GFX10-CU-NEXT:    s_clause 0x1
2835; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2836; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2837; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2838; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2839; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2840; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2841; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2842; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2843; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2844; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2845; GFX10-CU-NEXT:    s_endpgm
2846;
2847; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2848; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2849; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2850; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2851; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2852; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2853; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2854; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2855; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2856; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2857; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2858; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2859; SKIP-CACHE-INV-NEXT:    s_endpgm
2860;
2861; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2862; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2863; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2864; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2865; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2866; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2867; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2868; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2869; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2870;
2871; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2872; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2873; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2874; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2875; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2876; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2877; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2878; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2879; GFX90A-TGSPLIT-NEXT:    s_endpgm
2880;
2881; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2882; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2883; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2884; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2885; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2886; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2887; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2888; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2889; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2890;
2891; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2892; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2893; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2894; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2895; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2896; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2897; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2898; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2899; GFX940-TGSPLIT-NEXT:    s_endpgm
2900;
2901; GFX11-WGP-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2902; GFX11-WGP:       ; %bb.0: ; %entry
2903; GFX11-WGP-NEXT:    s_clause 0x1
2904; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2905; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2906; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2907; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2908; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2909; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2910; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2911; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2912; GFX11-WGP-NEXT:    s_endpgm
2913;
2914; GFX11-CU-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2915; GFX11-CU:       ; %bb.0: ; %entry
2916; GFX11-CU-NEXT:    s_clause 0x1
2917; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2918; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2919; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2920; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2921; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2922; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2923; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2924; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2925; GFX11-CU-NEXT:    s_endpgm
2926    i32* %out, i32 %in, i32 %old) {
2927entry:
2928  %gep = getelementptr i32, i32* %out, i32 4
2929  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
2930  ret void
2931}
2932
2933define amdgpu_kernel void @flat_singlethread_release_acquire_cmpxchg(
2934; GFX7-LABEL: flat_singlethread_release_acquire_cmpxchg:
2935; GFX7:       ; %bb.0: ; %entry
2936; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2937; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2938; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2939; GFX7-NEXT:    s_add_u32 s0, s0, 16
2940; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2941; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2942; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2943; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2944; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2945; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2946; GFX7-NEXT:    s_endpgm
2947;
2948; GFX10-WGP-LABEL: flat_singlethread_release_acquire_cmpxchg:
2949; GFX10-WGP:       ; %bb.0: ; %entry
2950; GFX10-WGP-NEXT:    s_clause 0x1
2951; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2952; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2953; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2954; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2955; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2956; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2957; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2958; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2959; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2960; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2961; GFX10-WGP-NEXT:    s_endpgm
2962;
2963; GFX10-CU-LABEL: flat_singlethread_release_acquire_cmpxchg:
2964; GFX10-CU:       ; %bb.0: ; %entry
2965; GFX10-CU-NEXT:    s_clause 0x1
2966; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2967; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2968; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2969; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2970; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2971; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2972; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2973; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2974; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2975; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2976; GFX10-CU-NEXT:    s_endpgm
2977;
2978; SKIP-CACHE-INV-LABEL: flat_singlethread_release_acquire_cmpxchg:
2979; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2980; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2981; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2982; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2983; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2984; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2985; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2986; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2987; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2988; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2989; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2990; SKIP-CACHE-INV-NEXT:    s_endpgm
2991;
2992; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_acquire_cmpxchg:
2993; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2994; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2995; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2996; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2997; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2998; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2999; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3000; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3001;
3002; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_acquire_cmpxchg:
3003; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3004; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3005; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3006; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3007; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3008; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3009; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3010; GFX90A-TGSPLIT-NEXT:    s_endpgm
3011;
3012; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_acquire_cmpxchg:
3013; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3014; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3015; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3016; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3017; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3018; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3019; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3020; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3021;
3022; GFX940-TGSPLIT-LABEL: flat_singlethread_release_acquire_cmpxchg:
3023; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3024; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3025; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3026; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3027; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3028; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3029; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3030; GFX940-TGSPLIT-NEXT:    s_endpgm
3031;
3032; GFX11-WGP-LABEL: flat_singlethread_release_acquire_cmpxchg:
3033; GFX11-WGP:       ; %bb.0: ; %entry
3034; GFX11-WGP-NEXT:    s_clause 0x1
3035; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3036; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3037; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3038; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3039; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3040; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3041; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3042; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3043; GFX11-WGP-NEXT:    s_endpgm
3044;
3045; GFX11-CU-LABEL: flat_singlethread_release_acquire_cmpxchg:
3046; GFX11-CU:       ; %bb.0: ; %entry
3047; GFX11-CU-NEXT:    s_clause 0x1
3048; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3049; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3050; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3051; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3052; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3053; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3054; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3055; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3056; GFX11-CU-NEXT:    s_endpgm
3057    i32* %out, i32 %in, i32 %old) {
3058entry:
3059  %gep = getelementptr i32, i32* %out, i32 4
3060  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
3061  ret void
3062}
3063
3064define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_cmpxchg(
3065; GFX7-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
3066; GFX7:       ; %bb.0: ; %entry
3067; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3068; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3069; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3070; GFX7-NEXT:    s_add_u32 s0, s0, 16
3071; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3072; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3073; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3074; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3075; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3076; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3077; GFX7-NEXT:    s_endpgm
3078;
3079; GFX10-WGP-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
3080; GFX10-WGP:       ; %bb.0: ; %entry
3081; GFX10-WGP-NEXT:    s_clause 0x1
3082; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3083; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3084; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3085; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3086; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3087; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3088; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3089; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3090; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3091; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3092; GFX10-WGP-NEXT:    s_endpgm
3093;
3094; GFX10-CU-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
3095; GFX10-CU:       ; %bb.0: ; %entry
3096; GFX10-CU-NEXT:    s_clause 0x1
3097; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3098; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3099; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3100; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3101; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3102; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3103; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3104; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3105; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3106; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3107; GFX10-CU-NEXT:    s_endpgm
3108;
3109; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
3110; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3111; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3112; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3113; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3114; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3115; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3116; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3117; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3118; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3119; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3120; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3121; SKIP-CACHE-INV-NEXT:    s_endpgm
3122;
3123; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
3124; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3125; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3126; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3127; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3128; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3129; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3130; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3131; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3132;
3133; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
3134; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3135; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3136; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3137; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3138; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3139; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3140; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3141; GFX90A-TGSPLIT-NEXT:    s_endpgm
3142;
3143; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
3144; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3145; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3146; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3147; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3148; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3149; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3150; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3151; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3152;
3153; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
3154; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3155; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3156; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3157; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3158; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3159; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3160; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3161; GFX940-TGSPLIT-NEXT:    s_endpgm
3162;
3163; GFX11-WGP-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
3164; GFX11-WGP:       ; %bb.0: ; %entry
3165; GFX11-WGP-NEXT:    s_clause 0x1
3166; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3167; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3168; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3169; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3170; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3171; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3172; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3173; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3174; GFX11-WGP-NEXT:    s_endpgm
3175;
3176; GFX11-CU-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
3177; GFX11-CU:       ; %bb.0: ; %entry
3178; GFX11-CU-NEXT:    s_clause 0x1
3179; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3180; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3181; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3182; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3183; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3184; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3185; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3186; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3187; GFX11-CU-NEXT:    s_endpgm
3188    i32* %out, i32 %in, i32 %old) {
3189entry:
3190  %gep = getelementptr i32, i32* %out, i32 4
3191  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
3192  ret void
3193}
3194
3195define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_cmpxchg(
3196; GFX7-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
3197; GFX7:       ; %bb.0: ; %entry
3198; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3199; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3200; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3201; GFX7-NEXT:    s_add_u32 s0, s0, 16
3202; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3203; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3204; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3205; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3206; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3207; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3208; GFX7-NEXT:    s_endpgm
3209;
3210; GFX10-WGP-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
3211; GFX10-WGP:       ; %bb.0: ; %entry
3212; GFX10-WGP-NEXT:    s_clause 0x1
3213; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3214; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3215; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3216; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3217; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3218; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3219; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3220; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3221; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3222; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3223; GFX10-WGP-NEXT:    s_endpgm
3224;
3225; GFX10-CU-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
3226; GFX10-CU:       ; %bb.0: ; %entry
3227; GFX10-CU-NEXT:    s_clause 0x1
3228; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3229; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3230; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3231; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3232; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3233; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3234; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3235; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3236; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3237; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3238; GFX10-CU-NEXT:    s_endpgm
3239;
3240; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
3241; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3242; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3243; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3244; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3245; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3246; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3247; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3248; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3249; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3250; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3251; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3252; SKIP-CACHE-INV-NEXT:    s_endpgm
3253;
3254; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
3255; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3256; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3257; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3258; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3259; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3260; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3261; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3262; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3263;
3264; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
3265; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3266; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3267; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3268; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3269; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3270; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3271; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3272; GFX90A-TGSPLIT-NEXT:    s_endpgm
3273;
3274; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
3275; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3276; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3277; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3278; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3279; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3280; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3281; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3282; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3283;
3284; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
3285; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3286; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3287; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3288; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3289; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3290; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3291; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3292; GFX940-TGSPLIT-NEXT:    s_endpgm
3293;
3294; GFX11-WGP-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
3295; GFX11-WGP:       ; %bb.0: ; %entry
3296; GFX11-WGP-NEXT:    s_clause 0x1
3297; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3298; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3299; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3300; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3301; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3302; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3303; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3304; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3305; GFX11-WGP-NEXT:    s_endpgm
3306;
3307; GFX11-CU-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
3308; GFX11-CU:       ; %bb.0: ; %entry
3309; GFX11-CU-NEXT:    s_clause 0x1
3310; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3311; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3312; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3313; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3314; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3315; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3316; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3317; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3318; GFX11-CU-NEXT:    s_endpgm
3319    i32* %out, i32 %in, i32 %old) {
3320entry:
3321  %gep = getelementptr i32, i32* %out, i32 4
3322  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
3323  ret void
3324}
3325
3326define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_cmpxchg(
3327; GFX7-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
3328; GFX7:       ; %bb.0: ; %entry
3329; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3330; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3331; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3332; GFX7-NEXT:    s_add_u32 s0, s0, 16
3333; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3334; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3335; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3336; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3337; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3338; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3339; GFX7-NEXT:    s_endpgm
3340;
3341; GFX10-WGP-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
3342; GFX10-WGP:       ; %bb.0: ; %entry
3343; GFX10-WGP-NEXT:    s_clause 0x1
3344; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3345; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3346; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3347; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3348; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3349; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3350; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3351; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3352; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3353; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3354; GFX10-WGP-NEXT:    s_endpgm
3355;
3356; GFX10-CU-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
3357; GFX10-CU:       ; %bb.0: ; %entry
3358; GFX10-CU-NEXT:    s_clause 0x1
3359; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3360; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3361; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3362; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3363; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3364; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3365; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3366; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3367; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3368; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3369; GFX10-CU-NEXT:    s_endpgm
3370;
3371; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
3372; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3373; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3374; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3375; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3376; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3377; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3378; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3379; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3380; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3381; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3382; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3383; SKIP-CACHE-INV-NEXT:    s_endpgm
3384;
3385; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
3386; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3387; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3388; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3389; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3390; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3391; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3392; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3393; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3394;
3395; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
3396; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3397; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3398; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3399; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3400; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3401; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3402; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3403; GFX90A-TGSPLIT-NEXT:    s_endpgm
3404;
3405; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
3406; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3407; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3408; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3409; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3410; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3411; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3412; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3413; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3414;
3415; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
3416; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3417; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3418; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3419; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3420; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3421; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3422; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3423; GFX940-TGSPLIT-NEXT:    s_endpgm
3424;
3425; GFX11-WGP-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
3426; GFX11-WGP:       ; %bb.0: ; %entry
3427; GFX11-WGP-NEXT:    s_clause 0x1
3428; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3429; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3430; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3431; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3432; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3433; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3434; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3435; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3436; GFX11-WGP-NEXT:    s_endpgm
3437;
3438; GFX11-CU-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
3439; GFX11-CU:       ; %bb.0: ; %entry
3440; GFX11-CU-NEXT:    s_clause 0x1
3441; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3442; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3443; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3444; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3445; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3446; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3447; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3448; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3449; GFX11-CU-NEXT:    s_endpgm
3450    i32* %out, i32 %in, i32 %old) {
3451entry:
3452  %gep = getelementptr i32, i32* %out, i32 4
3453  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic seq_cst
3454  ret void
3455}
3456
3457define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_cmpxchg(
3458; GFX7-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
3459; GFX7:       ; %bb.0: ; %entry
3460; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3461; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3462; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3463; GFX7-NEXT:    s_add_u32 s0, s0, 16
3464; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3465; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3466; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3467; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3468; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3469; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3470; GFX7-NEXT:    s_endpgm
3471;
3472; GFX10-WGP-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
3473; GFX10-WGP:       ; %bb.0: ; %entry
3474; GFX10-WGP-NEXT:    s_clause 0x1
3475; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3476; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3477; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3478; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3479; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3480; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3481; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3482; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3483; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3484; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3485; GFX10-WGP-NEXT:    s_endpgm
3486;
3487; GFX10-CU-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
3488; GFX10-CU:       ; %bb.0: ; %entry
3489; GFX10-CU-NEXT:    s_clause 0x1
3490; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3491; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3492; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3493; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3494; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3495; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3496; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3497; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3498; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3499; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3500; GFX10-CU-NEXT:    s_endpgm
3501;
3502; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
3503; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3504; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3505; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3506; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3507; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3508; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3509; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3510; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3511; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3512; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3513; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3514; SKIP-CACHE-INV-NEXT:    s_endpgm
3515;
3516; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
3517; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3518; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3519; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3520; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3521; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3522; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3523; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3524; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3525;
3526; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
3527; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3528; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3529; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3530; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3531; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3532; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3533; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3534; GFX90A-TGSPLIT-NEXT:    s_endpgm
3535;
3536; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
3537; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3538; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3539; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3540; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3541; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3542; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3543; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3544; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3545;
3546; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
3547; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3548; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3549; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3550; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3551; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3552; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3553; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3554; GFX940-TGSPLIT-NEXT:    s_endpgm
3555;
3556; GFX11-WGP-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
3557; GFX11-WGP:       ; %bb.0: ; %entry
3558; GFX11-WGP-NEXT:    s_clause 0x1
3559; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3560; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3561; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3562; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3563; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3564; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3565; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3566; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3567; GFX11-WGP-NEXT:    s_endpgm
3568;
3569; GFX11-CU-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
3570; GFX11-CU:       ; %bb.0: ; %entry
3571; GFX11-CU-NEXT:    s_clause 0x1
3572; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3573; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3574; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3575; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3576; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3577; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3578; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3579; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3580; GFX11-CU-NEXT:    s_endpgm
3581    i32* %out, i32 %in, i32 %old) {
3582entry:
3583  %gep = getelementptr i32, i32* %out, i32 4
3584  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire seq_cst
3585  ret void
3586}
3587
3588define amdgpu_kernel void @flat_singlethread_release_seq_cst_cmpxchg(
3589; GFX7-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
3590; GFX7:       ; %bb.0: ; %entry
3591; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3592; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3593; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3594; GFX7-NEXT:    s_add_u32 s0, s0, 16
3595; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3596; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3597; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3598; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3599; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3600; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3601; GFX7-NEXT:    s_endpgm
3602;
3603; GFX10-WGP-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
3604; GFX10-WGP:       ; %bb.0: ; %entry
3605; GFX10-WGP-NEXT:    s_clause 0x1
3606; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3607; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3608; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3609; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3610; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3611; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3612; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3613; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3614; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3615; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3616; GFX10-WGP-NEXT:    s_endpgm
3617;
3618; GFX10-CU-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
3619; GFX10-CU:       ; %bb.0: ; %entry
3620; GFX10-CU-NEXT:    s_clause 0x1
3621; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3622; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3623; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3624; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3625; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3626; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3627; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3628; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3629; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3630; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3631; GFX10-CU-NEXT:    s_endpgm
3632;
3633; SKIP-CACHE-INV-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
3634; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3635; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3636; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3637; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3638; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3639; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3640; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3641; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3642; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3643; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3644; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3645; SKIP-CACHE-INV-NEXT:    s_endpgm
3646;
3647; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
3648; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3649; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3650; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3651; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3652; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3653; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3654; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3655; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3656;
3657; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
3658; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3659; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3660; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3661; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3662; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3663; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3664; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3665; GFX90A-TGSPLIT-NEXT:    s_endpgm
3666;
3667; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
3668; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3669; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3670; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3671; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3672; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3673; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3674; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3675; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3676;
3677; GFX940-TGSPLIT-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
3678; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3679; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3680; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3681; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3682; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3683; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3684; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3685; GFX940-TGSPLIT-NEXT:    s_endpgm
3686;
3687; GFX11-WGP-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
3688; GFX11-WGP:       ; %bb.0: ; %entry
3689; GFX11-WGP-NEXT:    s_clause 0x1
3690; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3691; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3692; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3693; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3694; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3695; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3696; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3697; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3698; GFX11-WGP-NEXT:    s_endpgm
3699;
3700; GFX11-CU-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
3701; GFX11-CU:       ; %bb.0: ; %entry
3702; GFX11-CU-NEXT:    s_clause 0x1
3703; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3704; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3705; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3706; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3707; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3708; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3709; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3710; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3711; GFX11-CU-NEXT:    s_endpgm
3712    i32* %out, i32 %in, i32 %old) {
3713entry:
3714  %gep = getelementptr i32, i32* %out, i32 4
3715  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release seq_cst
3716  ret void
3717}
3718
3719define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_cmpxchg(
3720; GFX7-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3721; GFX7:       ; %bb.0: ; %entry
3722; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3723; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3724; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3725; GFX7-NEXT:    s_add_u32 s0, s0, 16
3726; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3727; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3728; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3729; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3730; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3731; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3732; GFX7-NEXT:    s_endpgm
3733;
3734; GFX10-WGP-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3735; GFX10-WGP:       ; %bb.0: ; %entry
3736; GFX10-WGP-NEXT:    s_clause 0x1
3737; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3738; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3739; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3740; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3741; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3742; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3743; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3744; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3745; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3746; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3747; GFX10-WGP-NEXT:    s_endpgm
3748;
3749; GFX10-CU-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3750; GFX10-CU:       ; %bb.0: ; %entry
3751; GFX10-CU-NEXT:    s_clause 0x1
3752; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3753; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3754; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3755; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3756; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3757; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3758; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3759; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3760; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3761; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3762; GFX10-CU-NEXT:    s_endpgm
3763;
3764; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3765; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3766; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3767; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3768; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3769; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3770; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3771; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3772; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3773; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3774; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3775; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3776; SKIP-CACHE-INV-NEXT:    s_endpgm
3777;
3778; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3779; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3780; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3781; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3782; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3783; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3784; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3785; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3786; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3787;
3788; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3789; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3790; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3791; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3792; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3793; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3794; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3795; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3796; GFX90A-TGSPLIT-NEXT:    s_endpgm
3797;
3798; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3799; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3800; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3801; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3802; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3803; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3804; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3805; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3806; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3807;
3808; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3809; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3810; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3811; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3812; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3813; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3814; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3815; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3816; GFX940-TGSPLIT-NEXT:    s_endpgm
3817;
3818; GFX11-WGP-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3819; GFX11-WGP:       ; %bb.0: ; %entry
3820; GFX11-WGP-NEXT:    s_clause 0x1
3821; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3822; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3823; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3824; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3825; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3826; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3827; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3828; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3829; GFX11-WGP-NEXT:    s_endpgm
3830;
3831; GFX11-CU-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3832; GFX11-CU:       ; %bb.0: ; %entry
3833; GFX11-CU-NEXT:    s_clause 0x1
3834; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3835; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3836; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3837; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3838; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3839; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3840; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3841; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3842; GFX11-CU-NEXT:    s_endpgm
3843    i32* %out, i32 %in, i32 %old) {
3844entry:
3845  %gep = getelementptr i32, i32* %out, i32 4
3846  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel seq_cst
3847  ret void
3848}
3849
3850define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_cmpxchg(
3851; GFX7-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3852; GFX7:       ; %bb.0: ; %entry
3853; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3854; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3855; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3856; GFX7-NEXT:    s_add_u32 s0, s0, 16
3857; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3858; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3859; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3860; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3861; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3862; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3863; GFX7-NEXT:    s_endpgm
3864;
3865; GFX10-WGP-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3866; GFX10-WGP:       ; %bb.0: ; %entry
3867; GFX10-WGP-NEXT:    s_clause 0x1
3868; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3869; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3870; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3871; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3872; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3873; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3874; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3875; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3876; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3877; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3878; GFX10-WGP-NEXT:    s_endpgm
3879;
3880; GFX10-CU-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3881; GFX10-CU:       ; %bb.0: ; %entry
3882; GFX10-CU-NEXT:    s_clause 0x1
3883; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3884; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3885; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3886; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3887; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3888; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3889; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3890; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3891; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3892; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3893; GFX10-CU-NEXT:    s_endpgm
3894;
3895; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3896; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3897; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3898; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3899; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3900; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3901; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3902; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3903; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3904; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3905; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3906; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3907; SKIP-CACHE-INV-NEXT:    s_endpgm
3908;
3909; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3910; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3911; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3912; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3913; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3914; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3915; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3916; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3917; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3918;
3919; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3920; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3921; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3922; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3923; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3924; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3925; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3926; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3927; GFX90A-TGSPLIT-NEXT:    s_endpgm
3928;
3929; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3930; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3931; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3932; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3933; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3934; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3935; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3936; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3937; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3938;
3939; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3940; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3941; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3942; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3943; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3944; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3945; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3946; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3947; GFX940-TGSPLIT-NEXT:    s_endpgm
3948;
3949; GFX11-WGP-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3950; GFX11-WGP:       ; %bb.0: ; %entry
3951; GFX11-WGP-NEXT:    s_clause 0x1
3952; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3953; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3954; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3955; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3956; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3957; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3958; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3959; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3960; GFX11-WGP-NEXT:    s_endpgm
3961;
3962; GFX11-CU-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3963; GFX11-CU:       ; %bb.0: ; %entry
3964; GFX11-CU-NEXT:    s_clause 0x1
3965; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3966; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3967; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3968; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3969; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3970; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3971; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3972; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3973; GFX11-CU-NEXT:    s_endpgm
3974    i32* %out, i32 %in, i32 %old) {
3975entry:
3976  %gep = getelementptr i32, i32* %out, i32 4
3977  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
3978  ret void
3979}
3980
3981define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_ret_cmpxchg(
3982; GFX7-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
3983; GFX7:       ; %bb.0: ; %entry
3984; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3985; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3986; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3987; GFX7-NEXT:    s_add_u32 s4, s0, 16
3988; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3989; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3990; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3991; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3992; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3993; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3994; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3995; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3996; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3997; GFX7-NEXT:    flat_store_dword v[0:1], v2
3998; GFX7-NEXT:    s_endpgm
3999;
4000; GFX10-WGP-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
4001; GFX10-WGP:       ; %bb.0: ; %entry
4002; GFX10-WGP-NEXT:    s_clause 0x1
4003; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4004; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4005; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4006; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4007; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4008; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4009; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4010; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4011; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4012; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4013; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4014; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4015; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4016; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4017; GFX10-WGP-NEXT:    s_endpgm
4018;
4019; GFX10-CU-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
4020; GFX10-CU:       ; %bb.0: ; %entry
4021; GFX10-CU-NEXT:    s_clause 0x1
4022; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4023; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4024; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4025; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4026; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4027; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4028; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4029; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4030; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4031; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4032; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4033; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4034; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4035; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4036; GFX10-CU-NEXT:    s_endpgm
4037;
4038; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
4039; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4040; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4041; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4042; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4043; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4044; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4045; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4046; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4047; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4048; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4049; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4050; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4051; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4052; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4053; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4054; SKIP-CACHE-INV-NEXT:    s_endpgm
4055;
4056; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
4057; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4058; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4059; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4060; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4061; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4062; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4063; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4064; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4065; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4066; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4067;
4068; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
4069; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4070; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4071; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4072; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4073; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4074; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4075; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4076; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4077; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4078; GFX90A-TGSPLIT-NEXT:    s_endpgm
4079;
4080; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
4081; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4082; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4083; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4084; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4085; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4086; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4087; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4088; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4089; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4090; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4091;
4092; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
4093; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4094; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4095; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4096; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4097; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4098; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4099; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4100; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4101; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4102; GFX940-TGSPLIT-NEXT:    s_endpgm
4103;
4104; GFX11-WGP-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
4105; GFX11-WGP:       ; %bb.0: ; %entry
4106; GFX11-WGP-NEXT:    s_clause 0x1
4107; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4108; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4109; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4110; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4111; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4112; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4113; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4114; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4115; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4116; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4117; GFX11-WGP-NEXT:    s_endpgm
4118;
4119; GFX11-CU-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
4120; GFX11-CU:       ; %bb.0: ; %entry
4121; GFX11-CU-NEXT:    s_clause 0x1
4122; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4123; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4124; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4125; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4126; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4127; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4128; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4129; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4130; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4131; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4132; GFX11-CU-NEXT:    s_endpgm
4133    i32* %out, i32 %in, i32 %old) {
4134entry:
4135  %gep = getelementptr i32, i32* %out, i32 4
4136  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
4137  %val0 = extractvalue { i32, i1 } %val, 0
4138  store i32 %val0, i32* %out, align 4
4139  ret void
4140}
4141
4142define amdgpu_kernel void @flat_singlethread_acquire_monotonic_ret_cmpxchg(
4143; GFX7-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
4144; GFX7:       ; %bb.0: ; %entry
4145; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4146; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4147; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4148; GFX7-NEXT:    s_add_u32 s4, s0, 16
4149; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4150; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4151; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4152; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4153; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4154; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4155; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4156; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4157; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4158; GFX7-NEXT:    flat_store_dword v[0:1], v2
4159; GFX7-NEXT:    s_endpgm
4160;
4161; GFX10-WGP-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
4162; GFX10-WGP:       ; %bb.0: ; %entry
4163; GFX10-WGP-NEXT:    s_clause 0x1
4164; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4165; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4166; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4167; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4168; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4169; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4170; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4171; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4172; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4173; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4174; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4175; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4176; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4177; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4178; GFX10-WGP-NEXT:    s_endpgm
4179;
4180; GFX10-CU-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
4181; GFX10-CU:       ; %bb.0: ; %entry
4182; GFX10-CU-NEXT:    s_clause 0x1
4183; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4184; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4185; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4186; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4187; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4188; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4189; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4190; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4191; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4192; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4193; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4194; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4195; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4196; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4197; GFX10-CU-NEXT:    s_endpgm
4198;
4199; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
4200; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4201; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4202; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4203; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4204; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4205; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4206; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4207; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4208; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4209; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4210; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4211; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4212; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4213; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4214; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4215; SKIP-CACHE-INV-NEXT:    s_endpgm
4216;
4217; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
4218; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4219; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4220; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4221; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4222; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4223; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4224; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4225; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4226; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4227; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4228;
4229; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
4230; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4231; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4232; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4233; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4234; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4235; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4236; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4237; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4238; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4239; GFX90A-TGSPLIT-NEXT:    s_endpgm
4240;
4241; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
4242; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4243; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4244; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4245; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4246; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4247; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4248; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4249; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4250; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4251; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4252;
4253; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
4254; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4255; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4256; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4257; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4258; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4259; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4260; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4261; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4262; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4263; GFX940-TGSPLIT-NEXT:    s_endpgm
4264;
4265; GFX11-WGP-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
4266; GFX11-WGP:       ; %bb.0: ; %entry
4267; GFX11-WGP-NEXT:    s_clause 0x1
4268; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4269; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4270; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4271; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4272; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4273; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4274; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4275; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4276; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4277; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4278; GFX11-WGP-NEXT:    s_endpgm
4279;
4280; GFX11-CU-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
4281; GFX11-CU:       ; %bb.0: ; %entry
4282; GFX11-CU-NEXT:    s_clause 0x1
4283; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4284; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4285; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4286; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4287; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4288; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4289; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4290; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4291; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4292; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4293; GFX11-CU-NEXT:    s_endpgm
4294    i32* %out, i32 %in, i32 %old) {
4295entry:
4296  %gep = getelementptr i32, i32* %out, i32 4
4297  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
4298  %val0 = extractvalue { i32, i1 } %val, 0
4299  store i32 %val0, i32* %out, align 4
4300  ret void
4301}
4302
4303define amdgpu_kernel void @flat_singlethread_release_monotonic_ret_cmpxchg(
4304; GFX7-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
4305; GFX7:       ; %bb.0: ; %entry
4306; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4307; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4308; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4309; GFX7-NEXT:    s_add_u32 s4, s0, 16
4310; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4311; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4312; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4313; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4314; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4315; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4316; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4317; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4318; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4319; GFX7-NEXT:    flat_store_dword v[0:1], v2
4320; GFX7-NEXT:    s_endpgm
4321;
4322; GFX10-WGP-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
4323; GFX10-WGP:       ; %bb.0: ; %entry
4324; GFX10-WGP-NEXT:    s_clause 0x1
4325; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4326; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4327; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4328; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4329; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4330; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4331; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4332; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4333; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4334; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4335; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4336; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4337; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4338; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4339; GFX10-WGP-NEXT:    s_endpgm
4340;
4341; GFX10-CU-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
4342; GFX10-CU:       ; %bb.0: ; %entry
4343; GFX10-CU-NEXT:    s_clause 0x1
4344; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4345; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4346; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4347; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4348; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4349; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4350; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4351; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4352; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4353; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4354; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4355; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4356; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4357; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4358; GFX10-CU-NEXT:    s_endpgm
4359;
4360; SKIP-CACHE-INV-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
4361; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4362; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4363; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4364; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4365; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4366; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4367; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4368; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4369; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4370; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4371; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4372; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4373; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4374; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4375; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4376; SKIP-CACHE-INV-NEXT:    s_endpgm
4377;
4378; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
4379; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4380; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4381; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4382; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4383; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4384; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4385; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4386; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4387; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4388; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4389;
4390; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
4391; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4392; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4393; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4394; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4395; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4396; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4397; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4398; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4399; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4400; GFX90A-TGSPLIT-NEXT:    s_endpgm
4401;
4402; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
4403; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4404; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4405; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4406; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4407; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4408; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4409; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4410; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4411; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4412; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4413;
4414; GFX940-TGSPLIT-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
4415; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4416; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4417; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4418; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4419; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4420; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4421; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4422; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4423; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4424; GFX940-TGSPLIT-NEXT:    s_endpgm
4425;
4426; GFX11-WGP-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
4427; GFX11-WGP:       ; %bb.0: ; %entry
4428; GFX11-WGP-NEXT:    s_clause 0x1
4429; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4430; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4431; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4432; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4433; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4434; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4435; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4436; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4437; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4438; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4439; GFX11-WGP-NEXT:    s_endpgm
4440;
4441; GFX11-CU-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
4442; GFX11-CU:       ; %bb.0: ; %entry
4443; GFX11-CU-NEXT:    s_clause 0x1
4444; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4445; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4446; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4447; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4448; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4449; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4450; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4451; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4452; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4453; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4454; GFX11-CU-NEXT:    s_endpgm
4455    i32* %out, i32 %in, i32 %old) {
4456entry:
4457  %gep = getelementptr i32, i32* %out, i32 4
4458  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
4459  %val0 = extractvalue { i32, i1 } %val, 0
4460  store i32 %val0, i32* %out, align 4
4461  ret void
4462}
4463
4464define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_ret_cmpxchg(
4465; GFX7-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
4466; GFX7:       ; %bb.0: ; %entry
4467; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4468; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4469; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4470; GFX7-NEXT:    s_add_u32 s4, s0, 16
4471; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4472; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4473; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4474; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4475; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4476; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4477; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4478; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4479; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4480; GFX7-NEXT:    flat_store_dword v[0:1], v2
4481; GFX7-NEXT:    s_endpgm
4482;
4483; GFX10-WGP-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
4484; GFX10-WGP:       ; %bb.0: ; %entry
4485; GFX10-WGP-NEXT:    s_clause 0x1
4486; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4487; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4488; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4489; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4490; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4491; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4492; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4493; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4494; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4495; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4496; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4497; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4498; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4499; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4500; GFX10-WGP-NEXT:    s_endpgm
4501;
4502; GFX10-CU-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
4503; GFX10-CU:       ; %bb.0: ; %entry
4504; GFX10-CU-NEXT:    s_clause 0x1
4505; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4506; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4507; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4508; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4509; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4510; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4511; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4512; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4513; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4514; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4515; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4516; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4517; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4518; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4519; GFX10-CU-NEXT:    s_endpgm
4520;
4521; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
4522; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4523; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4524; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4525; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4526; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4527; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4528; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4529; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4530; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4531; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4532; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4533; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4534; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4535; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4536; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4537; SKIP-CACHE-INV-NEXT:    s_endpgm
4538;
4539; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
4540; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4541; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4542; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4543; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4544; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4545; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4546; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4547; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4548; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4549; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4550;
4551; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
4552; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4553; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4554; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4555; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4556; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4557; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4558; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4559; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4560; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4561; GFX90A-TGSPLIT-NEXT:    s_endpgm
4562;
4563; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
4564; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4565; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4566; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4567; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4568; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4569; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4570; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4571; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4572; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4573; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4574;
4575; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
4576; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4577; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4578; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4579; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4580; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4581; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4582; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4583; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4584; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4585; GFX940-TGSPLIT-NEXT:    s_endpgm
4586;
4587; GFX11-WGP-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
4588; GFX11-WGP:       ; %bb.0: ; %entry
4589; GFX11-WGP-NEXT:    s_clause 0x1
4590; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4591; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4592; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4593; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4594; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4595; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4596; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4597; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4598; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4599; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4600; GFX11-WGP-NEXT:    s_endpgm
4601;
4602; GFX11-CU-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
4603; GFX11-CU:       ; %bb.0: ; %entry
4604; GFX11-CU-NEXT:    s_clause 0x1
4605; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4606; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4607; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4608; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4609; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4610; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4611; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4612; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4613; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4614; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4615; GFX11-CU-NEXT:    s_endpgm
4616    i32* %out, i32 %in, i32 %old) {
4617entry:
4618  %gep = getelementptr i32, i32* %out, i32 4
4619  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
4620  %val0 = extractvalue { i32, i1 } %val, 0
4621  store i32 %val0, i32* %out, align 4
4622  ret void
4623}
4624
4625define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_ret_cmpxchg(
4626; GFX7-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
4627; GFX7:       ; %bb.0: ; %entry
4628; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4629; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4630; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4631; GFX7-NEXT:    s_add_u32 s4, s0, 16
4632; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4633; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4634; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4635; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4636; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4637; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4638; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4639; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4640; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4641; GFX7-NEXT:    flat_store_dword v[0:1], v2
4642; GFX7-NEXT:    s_endpgm
4643;
4644; GFX10-WGP-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
4645; GFX10-WGP:       ; %bb.0: ; %entry
4646; GFX10-WGP-NEXT:    s_clause 0x1
4647; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4648; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4649; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4650; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4651; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4652; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4653; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4654; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4655; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4656; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4657; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4658; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4659; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4660; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4661; GFX10-WGP-NEXT:    s_endpgm
4662;
4663; GFX10-CU-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
4664; GFX10-CU:       ; %bb.0: ; %entry
4665; GFX10-CU-NEXT:    s_clause 0x1
4666; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4667; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4668; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4669; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4670; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4671; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4672; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4673; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4674; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4675; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4676; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4677; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4678; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4679; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4680; GFX10-CU-NEXT:    s_endpgm
4681;
4682; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
4683; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4684; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4685; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4686; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4687; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4688; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4689; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4690; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4691; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4692; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4693; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4694; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4695; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4696; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4697; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4698; SKIP-CACHE-INV-NEXT:    s_endpgm
4699;
4700; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
4701; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4702; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4703; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4704; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4705; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4706; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4707; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4708; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4709; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4710; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4711;
4712; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
4713; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4714; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4715; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4716; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4717; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4718; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4719; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4720; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4721; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4722; GFX90A-TGSPLIT-NEXT:    s_endpgm
4723;
4724; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
4725; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4726; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4727; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4728; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4729; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4730; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4731; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4732; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4733; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4734; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4735;
4736; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
4737; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4738; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4739; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4740; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4741; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4742; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4743; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4744; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4745; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4746; GFX940-TGSPLIT-NEXT:    s_endpgm
4747;
4748; GFX11-WGP-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
4749; GFX11-WGP:       ; %bb.0: ; %entry
4750; GFX11-WGP-NEXT:    s_clause 0x1
4751; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4752; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4753; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4754; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4755; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4756; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4757; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4758; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4759; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4760; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4761; GFX11-WGP-NEXT:    s_endpgm
4762;
4763; GFX11-CU-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
4764; GFX11-CU:       ; %bb.0: ; %entry
4765; GFX11-CU-NEXT:    s_clause 0x1
4766; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4767; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4768; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4769; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4770; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4771; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4772; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4773; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4774; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4775; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4776; GFX11-CU-NEXT:    s_endpgm
4777    i32* %out, i32 %in, i32 %old) {
4778entry:
4779  %gep = getelementptr i32, i32* %out, i32 4
4780  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
4781  %val0 = extractvalue { i32, i1 } %val, 0
4782  store i32 %val0, i32* %out, align 4
4783  ret void
4784}
4785
4786define amdgpu_kernel void @flat_singlethread_monotonic_acquire_ret_cmpxchg(
4787; GFX7-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
4788; GFX7:       ; %bb.0: ; %entry
4789; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4790; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4791; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4792; GFX7-NEXT:    s_add_u32 s4, s0, 16
4793; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4794; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4795; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4796; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4797; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4798; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4799; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4800; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4801; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4802; GFX7-NEXT:    flat_store_dword v[0:1], v2
4803; GFX7-NEXT:    s_endpgm
4804;
4805; GFX10-WGP-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
4806; GFX10-WGP:       ; %bb.0: ; %entry
4807; GFX10-WGP-NEXT:    s_clause 0x1
4808; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4809; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4810; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4811; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4812; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4813; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4814; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4815; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4816; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4817; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4818; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4819; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4820; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4821; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4822; GFX10-WGP-NEXT:    s_endpgm
4823;
4824; GFX10-CU-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
4825; GFX10-CU:       ; %bb.0: ; %entry
4826; GFX10-CU-NEXT:    s_clause 0x1
4827; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4828; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4829; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4830; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4831; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4832; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4833; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4834; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4835; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4836; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4837; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4838; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4839; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4840; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4841; GFX10-CU-NEXT:    s_endpgm
4842;
4843; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
4844; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4845; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4846; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4847; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4848; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4849; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4850; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4851; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4852; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4853; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4854; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4855; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4856; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4857; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4858; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4859; SKIP-CACHE-INV-NEXT:    s_endpgm
4860;
4861; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
4862; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4863; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4864; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4865; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4866; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4867; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4868; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4869; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4870; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4871; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4872;
4873; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
4874; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4875; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4876; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4877; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4878; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4879; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4880; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4881; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4882; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4883; GFX90A-TGSPLIT-NEXT:    s_endpgm
4884;
4885; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
4886; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4887; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4888; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4889; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4890; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4891; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4892; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4893; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4894; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4895; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4896;
4897; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
4898; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4899; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4900; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4901; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4902; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4903; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4904; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4905; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4906; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4907; GFX940-TGSPLIT-NEXT:    s_endpgm
4908;
4909; GFX11-WGP-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
4910; GFX11-WGP:       ; %bb.0: ; %entry
4911; GFX11-WGP-NEXT:    s_clause 0x1
4912; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4913; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4914; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4915; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4916; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4917; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4918; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4919; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4920; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4921; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4922; GFX11-WGP-NEXT:    s_endpgm
4923;
4924; GFX11-CU-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
4925; GFX11-CU:       ; %bb.0: ; %entry
4926; GFX11-CU-NEXT:    s_clause 0x1
4927; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4928; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4929; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4930; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4931; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4932; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4933; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4934; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4935; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4936; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4937; GFX11-CU-NEXT:    s_endpgm
4938    i32* %out, i32 %in, i32 %old) {
4939entry:
4940  %gep = getelementptr i32, i32* %out, i32 4
4941  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic acquire
4942  %val0 = extractvalue { i32, i1 } %val, 0
4943  store i32 %val0, i32* %out, align 4
4944  ret void
4945}
4946
4947define amdgpu_kernel void @flat_singlethread_acquire_acquire_ret_cmpxchg(
4948; GFX7-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
4949; GFX7:       ; %bb.0: ; %entry
4950; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4951; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4952; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4953; GFX7-NEXT:    s_add_u32 s4, s0, 16
4954; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4955; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4956; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4957; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4958; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4959; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4960; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4961; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4962; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4963; GFX7-NEXT:    flat_store_dword v[0:1], v2
4964; GFX7-NEXT:    s_endpgm
4965;
4966; GFX10-WGP-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
4967; GFX10-WGP:       ; %bb.0: ; %entry
4968; GFX10-WGP-NEXT:    s_clause 0x1
4969; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4970; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4971; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4972; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4973; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4974; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4975; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4976; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4977; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4978; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4979; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4980; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4981; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4982; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4983; GFX10-WGP-NEXT:    s_endpgm
4984;
4985; GFX10-CU-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
4986; GFX10-CU:       ; %bb.0: ; %entry
4987; GFX10-CU-NEXT:    s_clause 0x1
4988; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4989; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4990; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4991; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4992; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4993; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4994; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4995; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4996; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4997; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4998; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4999; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5000; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5001; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5002; GFX10-CU-NEXT:    s_endpgm
5003;
5004; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
5005; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5006; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5007; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5008; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5009; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5010; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5011; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5012; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5013; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5014; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5015; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5016; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5017; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5018; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5019; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5020; SKIP-CACHE-INV-NEXT:    s_endpgm
5021;
5022; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
5023; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5024; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5025; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5026; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5027; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5028; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5029; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5030; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5031; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5032; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5033;
5034; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
5035; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5036; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5037; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5038; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5039; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5040; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5041; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5042; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5043; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5044; GFX90A-TGSPLIT-NEXT:    s_endpgm
5045;
5046; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
5047; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5048; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5049; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5050; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5051; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5052; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5053; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5054; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5055; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5056; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5057;
5058; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
5059; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5060; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5061; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5062; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5063; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5064; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5065; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5066; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5067; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5068; GFX940-TGSPLIT-NEXT:    s_endpgm
5069;
5070; GFX11-WGP-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
5071; GFX11-WGP:       ; %bb.0: ; %entry
5072; GFX11-WGP-NEXT:    s_clause 0x1
5073; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5074; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5075; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5076; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5077; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5078; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5079; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5080; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5081; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5082; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5083; GFX11-WGP-NEXT:    s_endpgm
5084;
5085; GFX11-CU-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
5086; GFX11-CU:       ; %bb.0: ; %entry
5087; GFX11-CU-NEXT:    s_clause 0x1
5088; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5089; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5090; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5091; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5092; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5093; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5094; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5095; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5096; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5097; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5098; GFX11-CU-NEXT:    s_endpgm
5099    i32* %out, i32 %in, i32 %old) {
5100entry:
5101  %gep = getelementptr i32, i32* %out, i32 4
5102  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
5103  %val0 = extractvalue { i32, i1 } %val, 0
5104  store i32 %val0, i32* %out, align 4
5105  ret void
5106}
5107
5108define amdgpu_kernel void @flat_singlethread_release_acquire_ret_cmpxchg(
5109; GFX7-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
5110; GFX7:       ; %bb.0: ; %entry
5111; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5112; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5113; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5114; GFX7-NEXT:    s_add_u32 s4, s0, 16
5115; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5116; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5117; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5118; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5119; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5120; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5121; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5122; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5123; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5124; GFX7-NEXT:    flat_store_dword v[0:1], v2
5125; GFX7-NEXT:    s_endpgm
5126;
5127; GFX10-WGP-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
5128; GFX10-WGP:       ; %bb.0: ; %entry
5129; GFX10-WGP-NEXT:    s_clause 0x1
5130; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5131; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5132; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5133; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5134; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5135; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5136; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5137; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5138; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5139; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5140; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5141; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5142; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5143; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5144; GFX10-WGP-NEXT:    s_endpgm
5145;
5146; GFX10-CU-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
5147; GFX10-CU:       ; %bb.0: ; %entry
5148; GFX10-CU-NEXT:    s_clause 0x1
5149; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5150; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5151; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5152; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5153; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5154; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5155; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5156; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5157; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5158; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5159; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5160; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5161; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5162; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5163; GFX10-CU-NEXT:    s_endpgm
5164;
5165; SKIP-CACHE-INV-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
5166; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5167; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5168; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5169; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5170; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5171; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5172; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5173; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5174; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5175; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5176; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5177; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5178; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5179; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5180; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5181; SKIP-CACHE-INV-NEXT:    s_endpgm
5182;
5183; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
5184; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5185; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5186; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5187; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5188; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5189; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5190; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5191; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5192; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5193; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5194;
5195; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
5196; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5197; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5198; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5199; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5200; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5201; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5202; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5203; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5204; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5205; GFX90A-TGSPLIT-NEXT:    s_endpgm
5206;
5207; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
5208; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5209; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5210; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5211; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5212; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5213; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5214; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5215; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5216; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5217; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5218;
5219; GFX940-TGSPLIT-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
5220; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5221; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5222; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5223; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5224; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5225; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5226; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5227; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5228; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5229; GFX940-TGSPLIT-NEXT:    s_endpgm
5230;
5231; GFX11-WGP-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
5232; GFX11-WGP:       ; %bb.0: ; %entry
5233; GFX11-WGP-NEXT:    s_clause 0x1
5234; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5235; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5236; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5237; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5238; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5239; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5240; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5241; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5242; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5243; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5244; GFX11-WGP-NEXT:    s_endpgm
5245;
5246; GFX11-CU-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
5247; GFX11-CU:       ; %bb.0: ; %entry
5248; GFX11-CU-NEXT:    s_clause 0x1
5249; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5250; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5251; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5252; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5253; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5254; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5255; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5256; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5257; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5258; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5259; GFX11-CU-NEXT:    s_endpgm
5260    i32* %out, i32 %in, i32 %old) {
5261entry:
5262  %gep = getelementptr i32, i32* %out, i32 4
5263  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
5264  %val0 = extractvalue { i32, i1 } %val, 0
5265  store i32 %val0, i32* %out, align 4
5266  ret void
5267}
5268
5269define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_ret_cmpxchg(
5270; GFX7-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
5271; GFX7:       ; %bb.0: ; %entry
5272; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5273; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5274; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5275; GFX7-NEXT:    s_add_u32 s4, s0, 16
5276; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5277; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5278; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5279; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5280; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5281; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5282; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5283; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5284; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5285; GFX7-NEXT:    flat_store_dword v[0:1], v2
5286; GFX7-NEXT:    s_endpgm
5287;
5288; GFX10-WGP-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
5289; GFX10-WGP:       ; %bb.0: ; %entry
5290; GFX10-WGP-NEXT:    s_clause 0x1
5291; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5292; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5293; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5294; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5295; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5296; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5297; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5298; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5299; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5300; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5301; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5302; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5303; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5304; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5305; GFX10-WGP-NEXT:    s_endpgm
5306;
5307; GFX10-CU-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
5308; GFX10-CU:       ; %bb.0: ; %entry
5309; GFX10-CU-NEXT:    s_clause 0x1
5310; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5311; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5312; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5313; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5314; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5315; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5316; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5317; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5318; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5319; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5320; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5321; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5322; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5323; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5324; GFX10-CU-NEXT:    s_endpgm
5325;
5326; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
5327; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5328; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5329; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5330; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5331; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5332; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5333; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5334; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5335; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5336; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5337; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5338; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5339; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5340; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5341; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5342; SKIP-CACHE-INV-NEXT:    s_endpgm
5343;
5344; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
5345; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5346; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5347; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5348; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5349; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5350; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5351; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5352; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5353; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5354; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5355;
5356; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
5357; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5358; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5359; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5360; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5361; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5362; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5363; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5364; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5365; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5366; GFX90A-TGSPLIT-NEXT:    s_endpgm
5367;
5368; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
5369; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5370; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5371; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5372; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5373; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5374; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5375; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5376; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5377; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5378; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5379;
5380; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
5381; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5382; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5383; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5384; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5385; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5386; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5387; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5388; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5389; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5390; GFX940-TGSPLIT-NEXT:    s_endpgm
5391;
5392; GFX11-WGP-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
5393; GFX11-WGP:       ; %bb.0: ; %entry
5394; GFX11-WGP-NEXT:    s_clause 0x1
5395; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5396; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5397; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5398; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5399; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5400; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5401; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5402; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5403; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5404; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5405; GFX11-WGP-NEXT:    s_endpgm
5406;
5407; GFX11-CU-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
5408; GFX11-CU:       ; %bb.0: ; %entry
5409; GFX11-CU-NEXT:    s_clause 0x1
5410; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5411; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5412; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5413; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5414; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5415; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5416; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5417; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5418; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5419; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5420; GFX11-CU-NEXT:    s_endpgm
5421    i32* %out, i32 %in, i32 %old) {
5422entry:
5423  %gep = getelementptr i32, i32* %out, i32 4
5424  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
5425  %val0 = extractvalue { i32, i1 } %val, 0
5426  store i32 %val0, i32* %out, align 4
5427  ret void
5428}
5429
5430define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_ret_cmpxchg(
5431; GFX7-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
5432; GFX7:       ; %bb.0: ; %entry
5433; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5434; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5435; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5436; GFX7-NEXT:    s_add_u32 s4, s0, 16
5437; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5438; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5439; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5440; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5441; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5442; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5443; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5444; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5445; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5446; GFX7-NEXT:    flat_store_dword v[0:1], v2
5447; GFX7-NEXT:    s_endpgm
5448;
5449; GFX10-WGP-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
5450; GFX10-WGP:       ; %bb.0: ; %entry
5451; GFX10-WGP-NEXT:    s_clause 0x1
5452; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5453; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5454; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5455; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5456; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5457; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5458; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5459; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5460; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5461; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5462; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5463; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5464; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5465; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5466; GFX10-WGP-NEXT:    s_endpgm
5467;
5468; GFX10-CU-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
5469; GFX10-CU:       ; %bb.0: ; %entry
5470; GFX10-CU-NEXT:    s_clause 0x1
5471; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5472; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5473; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5474; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5475; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5476; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5477; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5478; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5479; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5480; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5481; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5482; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5483; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5484; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5485; GFX10-CU-NEXT:    s_endpgm
5486;
5487; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
5488; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5489; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5490; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5491; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5492; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5493; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5494; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5495; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5496; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5497; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5498; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5499; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5500; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5501; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5502; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5503; SKIP-CACHE-INV-NEXT:    s_endpgm
5504;
5505; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
5506; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5507; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5508; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5509; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5510; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5511; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5512; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5513; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5514; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5515; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5516;
5517; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
5518; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5519; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5520; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5521; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5522; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5523; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5524; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5525; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5526; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5527; GFX90A-TGSPLIT-NEXT:    s_endpgm
5528;
5529; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
5530; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5531; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5532; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5533; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5534; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5535; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5536; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5537; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5538; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5539; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5540;
5541; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
5542; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5543; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5544; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5545; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5546; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5547; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5548; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5549; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5550; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5551; GFX940-TGSPLIT-NEXT:    s_endpgm
5552;
5553; GFX11-WGP-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
5554; GFX11-WGP:       ; %bb.0: ; %entry
5555; GFX11-WGP-NEXT:    s_clause 0x1
5556; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5557; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5558; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5559; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5560; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5561; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5562; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5563; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5564; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5565; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5566; GFX11-WGP-NEXT:    s_endpgm
5567;
5568; GFX11-CU-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
5569; GFX11-CU:       ; %bb.0: ; %entry
5570; GFX11-CU-NEXT:    s_clause 0x1
5571; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5572; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5573; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5574; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5575; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5576; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5577; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5578; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5579; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5580; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5581; GFX11-CU-NEXT:    s_endpgm
5582    i32* %out, i32 %in, i32 %old) {
5583entry:
5584  %gep = getelementptr i32, i32* %out, i32 4
5585  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
5586  %val0 = extractvalue { i32, i1 } %val, 0
5587  store i32 %val0, i32* %out, align 4
5588  ret void
5589}
5590
5591define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_ret_cmpxchg(
5592; GFX7-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
5593; GFX7:       ; %bb.0: ; %entry
5594; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5595; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5596; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5597; GFX7-NEXT:    s_add_u32 s4, s0, 16
5598; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5599; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5600; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5601; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5602; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5603; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5604; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5605; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5606; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5607; GFX7-NEXT:    flat_store_dword v[0:1], v2
5608; GFX7-NEXT:    s_endpgm
5609;
5610; GFX10-WGP-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
5611; GFX10-WGP:       ; %bb.0: ; %entry
5612; GFX10-WGP-NEXT:    s_clause 0x1
5613; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5614; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5615; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5616; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5617; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5618; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5619; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5620; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5621; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5622; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5623; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5624; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5625; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5626; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5627; GFX10-WGP-NEXT:    s_endpgm
5628;
5629; GFX10-CU-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
5630; GFX10-CU:       ; %bb.0: ; %entry
5631; GFX10-CU-NEXT:    s_clause 0x1
5632; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5633; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5634; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5635; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5636; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5637; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5638; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5639; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5640; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5641; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5642; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5643; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5644; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5645; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5646; GFX10-CU-NEXT:    s_endpgm
5647;
5648; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
5649; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5650; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5651; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5652; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5653; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5654; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5655; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5656; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5657; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5658; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5659; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5660; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5661; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5662; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5663; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5664; SKIP-CACHE-INV-NEXT:    s_endpgm
5665;
5666; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
5667; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5668; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5669; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5670; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5671; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5672; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5673; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5674; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5675; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5676; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5677;
5678; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
5679; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5680; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5681; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5682; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5683; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5684; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5685; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5686; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5687; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5688; GFX90A-TGSPLIT-NEXT:    s_endpgm
5689;
5690; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
5691; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5692; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5693; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5694; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5695; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5696; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5697; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5698; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5699; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5700; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5701;
5702; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
5703; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5704; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5705; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5706; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5707; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5708; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5709; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5710; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5711; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5712; GFX940-TGSPLIT-NEXT:    s_endpgm
5713;
5714; GFX11-WGP-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
5715; GFX11-WGP:       ; %bb.0: ; %entry
5716; GFX11-WGP-NEXT:    s_clause 0x1
5717; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5718; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5719; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5720; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5721; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5722; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5723; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5724; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5725; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5726; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5727; GFX11-WGP-NEXT:    s_endpgm
5728;
5729; GFX11-CU-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
5730; GFX11-CU:       ; %bb.0: ; %entry
5731; GFX11-CU-NEXT:    s_clause 0x1
5732; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5733; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5734; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5735; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5736; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5737; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5738; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5739; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5740; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5741; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5742; GFX11-CU-NEXT:    s_endpgm
5743    i32* %out, i32 %in, i32 %old) {
5744entry:
5745  %gep = getelementptr i32, i32* %out, i32 4
5746  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic seq_cst
5747  %val0 = extractvalue { i32, i1 } %val, 0
5748  store i32 %val0, i32* %out, align 4
5749  ret void
5750}
5751
5752define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_ret_cmpxchg(
5753; GFX7-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
5754; GFX7:       ; %bb.0: ; %entry
5755; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5756; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5757; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5758; GFX7-NEXT:    s_add_u32 s4, s0, 16
5759; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5760; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5761; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5762; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5763; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5764; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5765; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5766; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5767; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5768; GFX7-NEXT:    flat_store_dword v[0:1], v2
5769; GFX7-NEXT:    s_endpgm
5770;
5771; GFX10-WGP-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
5772; GFX10-WGP:       ; %bb.0: ; %entry
5773; GFX10-WGP-NEXT:    s_clause 0x1
5774; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5775; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5776; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5777; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5778; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5779; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5780; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5781; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5782; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5783; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5784; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5785; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5786; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5787; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5788; GFX10-WGP-NEXT:    s_endpgm
5789;
5790; GFX10-CU-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
5791; GFX10-CU:       ; %bb.0: ; %entry
5792; GFX10-CU-NEXT:    s_clause 0x1
5793; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5794; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5795; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5796; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5797; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5798; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5799; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5800; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5801; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5802; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5803; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5804; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5805; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5806; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5807; GFX10-CU-NEXT:    s_endpgm
5808;
5809; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
5810; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5811; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5812; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5813; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5814; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5815; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5816; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5817; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5818; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5819; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5820; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5821; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5822; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5823; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5824; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5825; SKIP-CACHE-INV-NEXT:    s_endpgm
5826;
5827; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
5828; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5829; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5830; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5831; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5832; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5833; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5834; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5835; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5836; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5837; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5838;
5839; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
5840; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5841; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5842; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5843; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5844; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5845; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5846; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5847; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5848; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5849; GFX90A-TGSPLIT-NEXT:    s_endpgm
5850;
5851; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
5852; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5853; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5854; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5855; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5856; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5857; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5858; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5859; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5860; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5861; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5862;
5863; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
5864; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5865; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5866; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5867; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5868; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5869; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5870; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5871; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5872; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5873; GFX940-TGSPLIT-NEXT:    s_endpgm
5874;
5875; GFX11-WGP-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
5876; GFX11-WGP:       ; %bb.0: ; %entry
5877; GFX11-WGP-NEXT:    s_clause 0x1
5878; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5879; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5880; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5881; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5882; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5883; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5884; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5885; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5886; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5887; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5888; GFX11-WGP-NEXT:    s_endpgm
5889;
5890; GFX11-CU-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
5891; GFX11-CU:       ; %bb.0: ; %entry
5892; GFX11-CU-NEXT:    s_clause 0x1
5893; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5894; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5895; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5896; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5897; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5898; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5899; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5900; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5901; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5902; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5903; GFX11-CU-NEXT:    s_endpgm
5904    i32* %out, i32 %in, i32 %old) {
5905entry:
5906  %gep = getelementptr i32, i32* %out, i32 4
5907  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire seq_cst
5908  %val0 = extractvalue { i32, i1 } %val, 0
5909  store i32 %val0, i32* %out, align 4
5910  ret void
5911}
5912
5913define amdgpu_kernel void @flat_singlethread_release_seq_cst_ret_cmpxchg(
5914; GFX7-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
5915; GFX7:       ; %bb.0: ; %entry
5916; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5917; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5918; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5919; GFX7-NEXT:    s_add_u32 s4, s0, 16
5920; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5921; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5922; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5923; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5924; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5925; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5926; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5927; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5928; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5929; GFX7-NEXT:    flat_store_dword v[0:1], v2
5930; GFX7-NEXT:    s_endpgm
5931;
5932; GFX10-WGP-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
5933; GFX10-WGP:       ; %bb.0: ; %entry
5934; GFX10-WGP-NEXT:    s_clause 0x1
5935; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5936; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5937; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5938; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5939; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5940; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5941; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5942; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5943; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5944; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5945; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5946; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5947; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5948; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5949; GFX10-WGP-NEXT:    s_endpgm
5950;
5951; GFX10-CU-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
5952; GFX10-CU:       ; %bb.0: ; %entry
5953; GFX10-CU-NEXT:    s_clause 0x1
5954; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5955; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5956; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5957; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5958; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5959; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5960; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5961; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5962; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5963; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5964; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5965; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5966; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5967; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5968; GFX10-CU-NEXT:    s_endpgm
5969;
5970; SKIP-CACHE-INV-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
5971; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5972; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5973; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5974; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5975; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5976; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5977; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5978; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5979; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5980; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5981; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5982; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5983; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5984; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5985; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5986; SKIP-CACHE-INV-NEXT:    s_endpgm
5987;
5988; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
5989; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5990; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5991; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5992; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5993; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5994; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5995; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5996; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5997; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5998; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5999;
6000; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
6001; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6002; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6003; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6004; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6005; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6006; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6007; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6008; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6009; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6010; GFX90A-TGSPLIT-NEXT:    s_endpgm
6011;
6012; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
6013; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6014; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6015; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6016; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6017; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6018; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6019; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6020; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6021; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6022; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6023;
6024; GFX940-TGSPLIT-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
6025; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6026; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6027; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6028; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6029; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6030; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6031; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6032; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6033; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6034; GFX940-TGSPLIT-NEXT:    s_endpgm
6035;
6036; GFX11-WGP-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
6037; GFX11-WGP:       ; %bb.0: ; %entry
6038; GFX11-WGP-NEXT:    s_clause 0x1
6039; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6040; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6041; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6042; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6043; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
6044; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6045; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
6046; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6047; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6048; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6049; GFX11-WGP-NEXT:    s_endpgm
6050;
6051; GFX11-CU-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
6052; GFX11-CU:       ; %bb.0: ; %entry
6053; GFX11-CU-NEXT:    s_clause 0x1
6054; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6055; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6056; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6057; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6058; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
6059; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6060; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6061; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6062; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6063; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6064; GFX11-CU-NEXT:    s_endpgm
6065    i32* %out, i32 %in, i32 %old) {
6066entry:
6067  %gep = getelementptr i32, i32* %out, i32 4
6068  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release seq_cst
6069  %val0 = extractvalue { i32, i1 } %val, 0
6070  store i32 %val0, i32* %out, align 4
6071  ret void
6072}
6073
6074define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_ret_cmpxchg(
6075; GFX7-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6076; GFX7:       ; %bb.0: ; %entry
6077; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6078; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6079; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6080; GFX7-NEXT:    s_add_u32 s4, s0, 16
6081; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6082; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6083; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6084; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6085; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6086; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6087; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6088; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6089; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6090; GFX7-NEXT:    flat_store_dword v[0:1], v2
6091; GFX7-NEXT:    s_endpgm
6092;
6093; GFX10-WGP-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6094; GFX10-WGP:       ; %bb.0: ; %entry
6095; GFX10-WGP-NEXT:    s_clause 0x1
6096; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6097; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6098; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6099; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6100; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6101; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6102; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6103; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6104; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6105; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6106; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6107; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6108; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6109; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6110; GFX10-WGP-NEXT:    s_endpgm
6111;
6112; GFX10-CU-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6113; GFX10-CU:       ; %bb.0: ; %entry
6114; GFX10-CU-NEXT:    s_clause 0x1
6115; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6116; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6117; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6118; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6119; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6120; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6121; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6122; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6123; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6124; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6125; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6126; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6127; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6128; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6129; GFX10-CU-NEXT:    s_endpgm
6130;
6131; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6132; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6133; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6134; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6135; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6136; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6137; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6138; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6139; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6140; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6141; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6142; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6143; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6144; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6145; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6146; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6147; SKIP-CACHE-INV-NEXT:    s_endpgm
6148;
6149; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6150; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6151; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6152; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6153; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6154; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6155; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6156; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6157; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6158; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6159; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6160;
6161; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6162; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6163; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6164; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6165; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6166; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6167; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6168; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6169; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6170; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6171; GFX90A-TGSPLIT-NEXT:    s_endpgm
6172;
6173; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6174; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6175; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6176; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6177; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6178; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6179; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6180; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6181; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6182; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6183; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6184;
6185; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6186; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6187; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6188; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6189; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6190; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6191; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6192; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6193; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6194; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6195; GFX940-TGSPLIT-NEXT:    s_endpgm
6196;
6197; GFX11-WGP-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6198; GFX11-WGP:       ; %bb.0: ; %entry
6199; GFX11-WGP-NEXT:    s_clause 0x1
6200; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6201; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6202; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6203; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6204; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
6205; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6206; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
6207; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6208; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6209; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6210; GFX11-WGP-NEXT:    s_endpgm
6211;
6212; GFX11-CU-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6213; GFX11-CU:       ; %bb.0: ; %entry
6214; GFX11-CU-NEXT:    s_clause 0x1
6215; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6216; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6217; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6218; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6219; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
6220; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6221; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6222; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6223; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6224; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6225; GFX11-CU-NEXT:    s_endpgm
6226    i32* %out, i32 %in, i32 %old) {
6227entry:
6228  %gep = getelementptr i32, i32* %out, i32 4
6229  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel seq_cst
6230  %val0 = extractvalue { i32, i1 } %val, 0
6231  store i32 %val0, i32* %out, align 4
6232  ret void
6233}
6234
6235define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_ret_cmpxchg(
6236; GFX7-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6237; GFX7:       ; %bb.0: ; %entry
6238; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6239; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6240; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6241; GFX7-NEXT:    s_add_u32 s4, s0, 16
6242; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6243; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6244; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6245; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6246; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6247; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6248; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6249; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6250; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6251; GFX7-NEXT:    flat_store_dword v[0:1], v2
6252; GFX7-NEXT:    s_endpgm
6253;
6254; GFX10-WGP-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6255; GFX10-WGP:       ; %bb.0: ; %entry
6256; GFX10-WGP-NEXT:    s_clause 0x1
6257; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6258; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6259; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6260; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6261; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6262; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6263; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6264; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6265; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6266; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6267; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6268; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6269; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6270; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6271; GFX10-WGP-NEXT:    s_endpgm
6272;
6273; GFX10-CU-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6274; GFX10-CU:       ; %bb.0: ; %entry
6275; GFX10-CU-NEXT:    s_clause 0x1
6276; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6277; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6278; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6279; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6280; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6281; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6282; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6283; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6284; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6285; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6286; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6287; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6288; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6289; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6290; GFX10-CU-NEXT:    s_endpgm
6291;
6292; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6293; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6294; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6295; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6296; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6297; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6298; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6299; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6300; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6301; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6302; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6303; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6304; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6305; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6306; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6307; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6308; SKIP-CACHE-INV-NEXT:    s_endpgm
6309;
6310; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6311; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6312; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6313; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6314; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6315; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6316; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6317; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6318; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6319; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6320; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6321;
6322; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6323; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6324; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6325; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6326; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6327; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6328; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6329; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6330; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6331; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6332; GFX90A-TGSPLIT-NEXT:    s_endpgm
6333;
6334; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6335; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6336; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6337; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6338; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6339; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6340; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6341; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6342; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6343; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6344; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6345;
6346; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6347; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6348; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6349; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6350; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6351; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6352; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6353; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6354; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6355; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6356; GFX940-TGSPLIT-NEXT:    s_endpgm
6357;
6358; GFX11-WGP-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6359; GFX11-WGP:       ; %bb.0: ; %entry
6360; GFX11-WGP-NEXT:    s_clause 0x1
6361; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6362; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6363; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6364; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6365; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
6366; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6367; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
6368; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6369; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6370; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6371; GFX11-WGP-NEXT:    s_endpgm
6372;
6373; GFX11-CU-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6374; GFX11-CU:       ; %bb.0: ; %entry
6375; GFX11-CU-NEXT:    s_clause 0x1
6376; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6377; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6378; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6379; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6380; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
6381; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6382; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6383; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6384; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6385; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6386; GFX11-CU-NEXT:    s_endpgm
6387    i32* %out, i32 %in, i32 %old) {
6388entry:
6389  %gep = getelementptr i32, i32* %out, i32 4
6390  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
6391  %val0 = extractvalue { i32, i1 } %val, 0
6392  store i32 %val0, i32* %out, align 4
6393  ret void
6394}
6395
6396define amdgpu_kernel void @flat_singlethread_one_as_unordered_load(
6397; GFX7-LABEL: flat_singlethread_one_as_unordered_load:
6398; GFX7:       ; %bb.0: ; %entry
6399; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6400; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6401; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6402; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6403; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6404; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6405; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6406; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6407; GFX7-NEXT:    flat_store_dword v[0:1], v2
6408; GFX7-NEXT:    s_endpgm
6409;
6410; GFX10-WGP-LABEL: flat_singlethread_one_as_unordered_load:
6411; GFX10-WGP:       ; %bb.0: ; %entry
6412; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6413; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6414; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6415; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6416; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
6417; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6418; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6419; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6420; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6421; GFX10-WGP-NEXT:    s_endpgm
6422;
6423; GFX10-CU-LABEL: flat_singlethread_one_as_unordered_load:
6424; GFX10-CU:       ; %bb.0: ; %entry
6425; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6426; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6427; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6428; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6429; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6430; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6431; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6432; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6433; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6434; GFX10-CU-NEXT:    s_endpgm
6435;
6436; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_unordered_load:
6437; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6438; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6439; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6440; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6441; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6442; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
6443; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6444; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6445; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6446; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6447; SKIP-CACHE-INV-NEXT:    s_endpgm
6448;
6449; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_unordered_load:
6450; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6451; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6452; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6453; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6454; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6455; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6456; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6457; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6458; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6459; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6460; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6461;
6462; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_unordered_load:
6463; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6464; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6465; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6466; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6467; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6468; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6469; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6470; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6471; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6472; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6473; GFX90A-TGSPLIT-NEXT:    s_endpgm
6474;
6475; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_unordered_load:
6476; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6477; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6478; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6479; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6480; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6481; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6482; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6483; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6484; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6485; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6486; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6487;
6488; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_unordered_load:
6489; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6490; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6491; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6492; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6493; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6494; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6495; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6496; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6497; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6498; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6499; GFX940-TGSPLIT-NEXT:    s_endpgm
6500;
6501; GFX11-WGP-LABEL: flat_singlethread_one_as_unordered_load:
6502; GFX11-WGP:       ; %bb.0: ; %entry
6503; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6504; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6505; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
6506; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
6507; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
6508; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6509; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6510; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6511; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6512; GFX11-WGP-NEXT:    s_endpgm
6513;
6514; GFX11-CU-LABEL: flat_singlethread_one_as_unordered_load:
6515; GFX11-CU:       ; %bb.0: ; %entry
6516; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6517; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6518; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
6519; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
6520; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
6521; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6522; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6523; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6524; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6525; GFX11-CU-NEXT:    s_endpgm
6526    i32* %in, i32* %out) {
6527entry:
6528  %val = load atomic i32, i32* %in syncscope("singlethread-one-as") unordered, align 4
6529  store i32 %val, i32* %out
6530  ret void
6531}
6532
6533define amdgpu_kernel void @flat_singlethread_one_as_monotonic_load(
6534; GFX7-LABEL: flat_singlethread_one_as_monotonic_load:
6535; GFX7:       ; %bb.0: ; %entry
6536; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6537; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6538; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6539; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6540; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6541; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6542; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6543; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6544; GFX7-NEXT:    flat_store_dword v[0:1], v2
6545; GFX7-NEXT:    s_endpgm
6546;
6547; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_load:
6548; GFX10-WGP:       ; %bb.0: ; %entry
6549; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6550; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6551; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6552; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6553; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
6554; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6555; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6556; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6557; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6558; GFX10-WGP-NEXT:    s_endpgm
6559;
6560; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_load:
6561; GFX10-CU:       ; %bb.0: ; %entry
6562; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6563; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6564; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6565; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6566; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6567; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6568; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6569; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6570; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6571; GFX10-CU-NEXT:    s_endpgm
6572;
6573; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_load:
6574; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6575; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6576; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6577; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6578; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6579; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
6580; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6581; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6582; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6583; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6584; SKIP-CACHE-INV-NEXT:    s_endpgm
6585;
6586; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_load:
6587; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6588; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6589; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6590; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6591; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6592; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6593; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6594; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6595; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6596; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6597; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6598;
6599; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_load:
6600; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6601; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6602; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6603; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6604; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6605; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6606; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6607; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6608; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6609; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6610; GFX90A-TGSPLIT-NEXT:    s_endpgm
6611;
6612; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_load:
6613; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6614; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6615; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6616; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6617; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6618; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6619; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6620; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6621; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6622; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6623; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6624;
6625; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_load:
6626; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6627; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6628; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6629; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6630; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6631; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6632; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6633; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6634; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6635; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6636; GFX940-TGSPLIT-NEXT:    s_endpgm
6637;
6638; GFX11-WGP-LABEL: flat_singlethread_one_as_monotonic_load:
6639; GFX11-WGP:       ; %bb.0: ; %entry
6640; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6641; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6642; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
6643; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
6644; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
6645; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6646; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6647; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6648; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6649; GFX11-WGP-NEXT:    s_endpgm
6650;
6651; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_load:
6652; GFX11-CU:       ; %bb.0: ; %entry
6653; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6654; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6655; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
6656; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
6657; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
6658; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6659; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6660; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6661; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6662; GFX11-CU-NEXT:    s_endpgm
6663    i32* %in, i32* %out) {
6664entry:
6665  %val = load atomic i32, i32* %in syncscope("singlethread-one-as") monotonic, align 4
6666  store i32 %val, i32* %out
6667  ret void
6668}
6669
6670define amdgpu_kernel void @flat_singlethread_one_as_acquire_load(
6671; GFX7-LABEL: flat_singlethread_one_as_acquire_load:
6672; GFX7:       ; %bb.0: ; %entry
6673; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6674; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6675; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6676; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6677; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6678; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6679; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6680; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6681; GFX7-NEXT:    flat_store_dword v[0:1], v2
6682; GFX7-NEXT:    s_endpgm
6683;
6684; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_load:
6685; GFX10-WGP:       ; %bb.0: ; %entry
6686; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6687; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6688; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6689; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6690; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
6691; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6692; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6693; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6694; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6695; GFX10-WGP-NEXT:    s_endpgm
6696;
6697; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_load:
6698; GFX10-CU:       ; %bb.0: ; %entry
6699; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6700; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6701; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6702; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6703; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6704; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6705; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6706; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6707; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6708; GFX10-CU-NEXT:    s_endpgm
6709;
6710; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_load:
6711; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6712; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6713; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6714; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6715; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6716; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
6717; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6718; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6719; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6720; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6721; SKIP-CACHE-INV-NEXT:    s_endpgm
6722;
6723; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_load:
6724; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6725; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6726; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6727; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6728; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6729; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6730; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6731; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6732; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6733; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6734; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6735;
6736; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_load:
6737; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6738; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6739; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6740; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6741; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6742; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6743; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6744; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6745; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6746; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6747; GFX90A-TGSPLIT-NEXT:    s_endpgm
6748;
6749; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_load:
6750; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6751; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6752; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6753; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6754; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6755; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6756; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6757; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6758; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6759; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6760; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6761;
6762; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_load:
6763; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6764; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6765; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6766; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6767; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6768; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6769; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6770; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6771; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6772; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6773; GFX940-TGSPLIT-NEXT:    s_endpgm
6774;
6775; GFX11-WGP-LABEL: flat_singlethread_one_as_acquire_load:
6776; GFX11-WGP:       ; %bb.0: ; %entry
6777; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6778; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6779; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
6780; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
6781; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
6782; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6783; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6784; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6785; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6786; GFX11-WGP-NEXT:    s_endpgm
6787;
6788; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_load:
6789; GFX11-CU:       ; %bb.0: ; %entry
6790; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6791; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6792; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
6793; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
6794; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
6795; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6796; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6797; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6798; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6799; GFX11-CU-NEXT:    s_endpgm
6800    i32* %in, i32* %out) {
6801entry:
6802  %val = load atomic i32, i32* %in syncscope("singlethread-one-as") acquire, align 4
6803  store i32 %val, i32* %out
6804  ret void
6805}
6806
6807define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_load(
6808; GFX7-LABEL: flat_singlethread_one_as_seq_cst_load:
6809; GFX7:       ; %bb.0: ; %entry
6810; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6811; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6812; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6813; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6814; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6815; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6816; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6817; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6818; GFX7-NEXT:    flat_store_dword v[0:1], v2
6819; GFX7-NEXT:    s_endpgm
6820;
6821; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_load:
6822; GFX10-WGP:       ; %bb.0: ; %entry
6823; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6824; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6825; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6826; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6827; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
6828; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6829; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6830; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6831; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6832; GFX10-WGP-NEXT:    s_endpgm
6833;
6834; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_load:
6835; GFX10-CU:       ; %bb.0: ; %entry
6836; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6837; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6838; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6839; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6840; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6841; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6842; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6843; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6844; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6845; GFX10-CU-NEXT:    s_endpgm
6846;
6847; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_load:
6848; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6849; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6850; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6851; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6852; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6853; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
6854; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6855; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6856; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6857; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6858; SKIP-CACHE-INV-NEXT:    s_endpgm
6859;
6860; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_load:
6861; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6862; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6863; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6864; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6865; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6866; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6867; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6868; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6869; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6870; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6871; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6872;
6873; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_load:
6874; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6875; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6876; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6877; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6878; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6879; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6880; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6881; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6882; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6883; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6884; GFX90A-TGSPLIT-NEXT:    s_endpgm
6885;
6886; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_load:
6887; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6888; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6889; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6890; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6891; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6892; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6893; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6894; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6895; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6896; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6897; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6898;
6899; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_load:
6900; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6901; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6902; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6903; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6904; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6905; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6906; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6907; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6908; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6909; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6910; GFX940-TGSPLIT-NEXT:    s_endpgm
6911;
6912; GFX11-WGP-LABEL: flat_singlethread_one_as_seq_cst_load:
6913; GFX11-WGP:       ; %bb.0: ; %entry
6914; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6915; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6916; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
6917; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
6918; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
6919; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6920; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6921; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6922; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6923; GFX11-WGP-NEXT:    s_endpgm
6924;
6925; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_load:
6926; GFX11-CU:       ; %bb.0: ; %entry
6927; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6928; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6929; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
6930; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
6931; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
6932; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6933; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6934; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6935; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6936; GFX11-CU-NEXT:    s_endpgm
6937    i32* %in, i32* %out) {
6938entry:
6939  %val = load atomic i32, i32* %in syncscope("singlethread-one-as") seq_cst, align 4
6940  store i32 %val, i32* %out
6941  ret void
6942}
6943
6944define amdgpu_kernel void @flat_singlethread_one_as_unordered_store(
6945; GFX7-LABEL: flat_singlethread_one_as_unordered_store:
6946; GFX7:       ; %bb.0: ; %entry
6947; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
6948; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
6949; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6950; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6951; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6952; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6953; GFX7-NEXT:    flat_store_dword v[0:1], v2
6954; GFX7-NEXT:    s_endpgm
6955;
6956; GFX10-WGP-LABEL: flat_singlethread_one_as_unordered_store:
6957; GFX10-WGP:       ; %bb.0: ; %entry
6958; GFX10-WGP-NEXT:    s_clause 0x1
6959; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6960; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
6961; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6962; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6963; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6964; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6965; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6966; GFX10-WGP-NEXT:    s_endpgm
6967;
6968; GFX10-CU-LABEL: flat_singlethread_one_as_unordered_store:
6969; GFX10-CU:       ; %bb.0: ; %entry
6970; GFX10-CU-NEXT:    s_clause 0x1
6971; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6972; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
6973; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6974; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6975; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6976; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6977; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6978; GFX10-CU-NEXT:    s_endpgm
6979;
6980; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_unordered_store:
6981; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6982; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
6983; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
6984; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6985; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6986; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6987; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6988; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6989; SKIP-CACHE-INV-NEXT:    s_endpgm
6990;
6991; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_unordered_store:
6992; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6993; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6994; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6995; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6996; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6997; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6998; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6999; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7000;
7001; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_unordered_store:
7002; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7003; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7004; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7005; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7006; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7007; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7008; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7009; GFX90A-TGSPLIT-NEXT:    s_endpgm
7010;
7011; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_unordered_store:
7012; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7013; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7014; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7015; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7016; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7017; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7018; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7019; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7020;
7021; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_unordered_store:
7022; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7023; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7024; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7025; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7026; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7027; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7028; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7029; GFX940-TGSPLIT-NEXT:    s_endpgm
7030;
7031; GFX11-WGP-LABEL: flat_singlethread_one_as_unordered_store:
7032; GFX11-WGP:       ; %bb.0: ; %entry
7033; GFX11-WGP-NEXT:    s_clause 0x1
7034; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7035; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
7036; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7037; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7038; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7039; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7040; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7041; GFX11-WGP-NEXT:    s_endpgm
7042;
7043; GFX11-CU-LABEL: flat_singlethread_one_as_unordered_store:
7044; GFX11-CU:       ; %bb.0: ; %entry
7045; GFX11-CU-NEXT:    s_clause 0x1
7046; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7047; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
7048; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7049; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7050; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7051; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7052; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7053; GFX11-CU-NEXT:    s_endpgm
7054    i32 %in, i32* %out) {
7055entry:
7056  store atomic i32 %in, i32* %out syncscope("singlethread-one-as") unordered, align 4
7057  ret void
7058}
7059
7060define amdgpu_kernel void @flat_singlethread_one_as_monotonic_store(
7061; GFX7-LABEL: flat_singlethread_one_as_monotonic_store:
7062; GFX7:       ; %bb.0: ; %entry
7063; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7064; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7065; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7066; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7067; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7068; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7069; GFX7-NEXT:    flat_store_dword v[0:1], v2
7070; GFX7-NEXT:    s_endpgm
7071;
7072; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_store:
7073; GFX10-WGP:       ; %bb.0: ; %entry
7074; GFX10-WGP-NEXT:    s_clause 0x1
7075; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7076; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7077; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7078; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7079; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7080; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7081; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7082; GFX10-WGP-NEXT:    s_endpgm
7083;
7084; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_store:
7085; GFX10-CU:       ; %bb.0: ; %entry
7086; GFX10-CU-NEXT:    s_clause 0x1
7087; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7088; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7089; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7090; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7091; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7092; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7093; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7094; GFX10-CU-NEXT:    s_endpgm
7095;
7096; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_store:
7097; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7098; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
7099; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
7100; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7101; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7102; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7103; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7104; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7105; SKIP-CACHE-INV-NEXT:    s_endpgm
7106;
7107; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_store:
7108; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7109; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7110; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7111; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7112; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7113; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7114; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7115; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7116;
7117; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_store:
7118; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7119; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7120; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7121; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7122; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7123; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7124; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7125; GFX90A-TGSPLIT-NEXT:    s_endpgm
7126;
7127; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_store:
7128; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7129; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7130; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7131; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7132; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7133; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7134; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7135; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7136;
7137; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_store:
7138; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7139; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7140; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7141; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7142; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7143; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7144; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7145; GFX940-TGSPLIT-NEXT:    s_endpgm
7146;
7147; GFX11-WGP-LABEL: flat_singlethread_one_as_monotonic_store:
7148; GFX11-WGP:       ; %bb.0: ; %entry
7149; GFX11-WGP-NEXT:    s_clause 0x1
7150; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7151; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
7152; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7153; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7154; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7155; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7156; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7157; GFX11-WGP-NEXT:    s_endpgm
7158;
7159; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_store:
7160; GFX11-CU:       ; %bb.0: ; %entry
7161; GFX11-CU-NEXT:    s_clause 0x1
7162; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7163; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
7164; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7165; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7166; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7167; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7168; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7169; GFX11-CU-NEXT:    s_endpgm
7170    i32 %in, i32* %out) {
7171entry:
7172  store atomic i32 %in, i32* %out syncscope("singlethread-one-as") monotonic, align 4
7173  ret void
7174}
7175
7176define amdgpu_kernel void @flat_singlethread_one_as_release_store(
7177; GFX7-LABEL: flat_singlethread_one_as_release_store:
7178; GFX7:       ; %bb.0: ; %entry
7179; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7180; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7181; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7182; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7183; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7184; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7185; GFX7-NEXT:    flat_store_dword v[0:1], v2
7186; GFX7-NEXT:    s_endpgm
7187;
7188; GFX10-WGP-LABEL: flat_singlethread_one_as_release_store:
7189; GFX10-WGP:       ; %bb.0: ; %entry
7190; GFX10-WGP-NEXT:    s_clause 0x1
7191; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7192; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7193; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7194; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7195; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7196; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7197; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7198; GFX10-WGP-NEXT:    s_endpgm
7199;
7200; GFX10-CU-LABEL: flat_singlethread_one_as_release_store:
7201; GFX10-CU:       ; %bb.0: ; %entry
7202; GFX10-CU-NEXT:    s_clause 0x1
7203; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7204; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7205; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7206; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7207; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7208; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7209; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7210; GFX10-CU-NEXT:    s_endpgm
7211;
7212; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_store:
7213; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7214; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
7215; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
7216; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7217; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7218; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7219; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7220; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7221; SKIP-CACHE-INV-NEXT:    s_endpgm
7222;
7223; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_store:
7224; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7225; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7226; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7227; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7228; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7229; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7230; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7231; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7232;
7233; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_store:
7234; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7235; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7236; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7237; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7238; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7239; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7240; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7241; GFX90A-TGSPLIT-NEXT:    s_endpgm
7242;
7243; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_store:
7244; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7245; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7246; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7247; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7248; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7249; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7250; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7251; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7252;
7253; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_store:
7254; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7255; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7256; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7257; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7258; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7259; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7260; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7261; GFX940-TGSPLIT-NEXT:    s_endpgm
7262;
7263; GFX11-WGP-LABEL: flat_singlethread_one_as_release_store:
7264; GFX11-WGP:       ; %bb.0: ; %entry
7265; GFX11-WGP-NEXT:    s_clause 0x1
7266; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7267; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
7268; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7269; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7270; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7271; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7272; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7273; GFX11-WGP-NEXT:    s_endpgm
7274;
7275; GFX11-CU-LABEL: flat_singlethread_one_as_release_store:
7276; GFX11-CU:       ; %bb.0: ; %entry
7277; GFX11-CU-NEXT:    s_clause 0x1
7278; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7279; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
7280; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7281; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7282; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7283; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7284; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7285; GFX11-CU-NEXT:    s_endpgm
7286    i32 %in, i32* %out) {
7287entry:
7288  store atomic i32 %in, i32* %out syncscope("singlethread-one-as") release, align 4
7289  ret void
7290}
7291
7292define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_store(
7293; GFX7-LABEL: flat_singlethread_one_as_seq_cst_store:
7294; GFX7:       ; %bb.0: ; %entry
7295; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7296; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7297; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7298; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7299; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7300; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7301; GFX7-NEXT:    flat_store_dword v[0:1], v2
7302; GFX7-NEXT:    s_endpgm
7303;
7304; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_store:
7305; GFX10-WGP:       ; %bb.0: ; %entry
7306; GFX10-WGP-NEXT:    s_clause 0x1
7307; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7308; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7309; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7310; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7311; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7312; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7313; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7314; GFX10-WGP-NEXT:    s_endpgm
7315;
7316; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_store:
7317; GFX10-CU:       ; %bb.0: ; %entry
7318; GFX10-CU-NEXT:    s_clause 0x1
7319; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7320; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7321; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7322; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7323; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7324; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7325; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7326; GFX10-CU-NEXT:    s_endpgm
7327;
7328; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_store:
7329; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7330; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
7331; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
7332; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7333; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7334; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7335; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7336; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7337; SKIP-CACHE-INV-NEXT:    s_endpgm
7338;
7339; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_store:
7340; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7341; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7342; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7343; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7344; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7345; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7346; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7347; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7348;
7349; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_store:
7350; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7351; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7352; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7353; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7354; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7355; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7356; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7357; GFX90A-TGSPLIT-NEXT:    s_endpgm
7358;
7359; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_store:
7360; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7361; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7362; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7363; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7364; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7365; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7366; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7367; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7368;
7369; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_store:
7370; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7371; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7372; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7373; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7374; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7375; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7376; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7377; GFX940-TGSPLIT-NEXT:    s_endpgm
7378;
7379; GFX11-WGP-LABEL: flat_singlethread_one_as_seq_cst_store:
7380; GFX11-WGP:       ; %bb.0: ; %entry
7381; GFX11-WGP-NEXT:    s_clause 0x1
7382; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7383; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
7384; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7385; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7386; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7387; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7388; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7389; GFX11-WGP-NEXT:    s_endpgm
7390;
7391; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_store:
7392; GFX11-CU:       ; %bb.0: ; %entry
7393; GFX11-CU-NEXT:    s_clause 0x1
7394; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
7395; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
7396; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7397; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7398; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7399; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7400; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7401; GFX11-CU-NEXT:    s_endpgm
7402    i32 %in, i32* %out) {
7403entry:
7404  store atomic i32 %in, i32* %out syncscope("singlethread-one-as") seq_cst, align 4
7405  ret void
7406}
7407
7408define amdgpu_kernel void @flat_singlethread_one_as_monotonic_atomicrmw(
7409; GFX7-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
7410; GFX7:       ; %bb.0: ; %entry
7411; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7412; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7413; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7414; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7415; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7416; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7417; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7418; GFX7-NEXT:    s_endpgm
7419;
7420; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
7421; GFX10-WGP:       ; %bb.0: ; %entry
7422; GFX10-WGP-NEXT:    s_clause 0x1
7423; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7424; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7425; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7426; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7427; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7428; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7429; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7430; GFX10-WGP-NEXT:    s_endpgm
7431;
7432; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
7433; GFX10-CU:       ; %bb.0: ; %entry
7434; GFX10-CU-NEXT:    s_clause 0x1
7435; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7436; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7437; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7438; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7439; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7440; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7441; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7442; GFX10-CU-NEXT:    s_endpgm
7443;
7444; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
7445; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7446; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7447; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7448; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7449; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7450; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7451; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7452; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7453; SKIP-CACHE-INV-NEXT:    s_endpgm
7454;
7455; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
7456; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7457; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7458; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7459; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7460; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7461; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7462; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7463; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7464;
7465; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
7466; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7467; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7468; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7469; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7470; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7471; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7472; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7473; GFX90A-TGSPLIT-NEXT:    s_endpgm
7474;
7475; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
7476; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7477; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7478; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7479; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7480; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7481; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7482; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7483; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7484;
7485; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
7486; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7487; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7488; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7489; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7490; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7491; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7492; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7493; GFX940-TGSPLIT-NEXT:    s_endpgm
7494;
7495; GFX11-WGP-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
7496; GFX11-WGP:       ; %bb.0: ; %entry
7497; GFX11-WGP-NEXT:    s_clause 0x1
7498; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7499; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
7500; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7501; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7502; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7503; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7504; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7505; GFX11-WGP-NEXT:    s_endpgm
7506;
7507; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
7508; GFX11-CU:       ; %bb.0: ; %entry
7509; GFX11-CU-NEXT:    s_clause 0x1
7510; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7511; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7512; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7513; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7514; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7515; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7516; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7517; GFX11-CU-NEXT:    s_endpgm
7518    i32* %out, i32 %in) {
7519entry:
7520  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") monotonic
7521  ret void
7522}
7523
7524define amdgpu_kernel void @flat_singlethread_one_as_acquire_atomicrmw(
7525; GFX7-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
7526; GFX7:       ; %bb.0: ; %entry
7527; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7528; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7529; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7530; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7531; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7532; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7533; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7534; GFX7-NEXT:    s_endpgm
7535;
7536; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
7537; GFX10-WGP:       ; %bb.0: ; %entry
7538; GFX10-WGP-NEXT:    s_clause 0x1
7539; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7540; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7541; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7542; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7543; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7544; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7545; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7546; GFX10-WGP-NEXT:    s_endpgm
7547;
7548; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
7549; GFX10-CU:       ; %bb.0: ; %entry
7550; GFX10-CU-NEXT:    s_clause 0x1
7551; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7552; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7553; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7554; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7555; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7556; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7557; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7558; GFX10-CU-NEXT:    s_endpgm
7559;
7560; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
7561; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7562; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7563; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7564; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7565; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7566; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7567; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7568; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7569; SKIP-CACHE-INV-NEXT:    s_endpgm
7570;
7571; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
7572; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7573; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7574; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7575; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7576; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7577; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7578; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7579; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7580;
7581; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
7582; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7583; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7584; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7585; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7586; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7587; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7588; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7589; GFX90A-TGSPLIT-NEXT:    s_endpgm
7590;
7591; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
7592; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7593; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7594; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7595; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7596; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7597; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7598; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7599; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7600;
7601; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
7602; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7603; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7604; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7605; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7606; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7607; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7608; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7609; GFX940-TGSPLIT-NEXT:    s_endpgm
7610;
7611; GFX11-WGP-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
7612; GFX11-WGP:       ; %bb.0: ; %entry
7613; GFX11-WGP-NEXT:    s_clause 0x1
7614; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7615; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
7616; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7617; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7618; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7619; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7620; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7621; GFX11-WGP-NEXT:    s_endpgm
7622;
7623; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
7624; GFX11-CU:       ; %bb.0: ; %entry
7625; GFX11-CU-NEXT:    s_clause 0x1
7626; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7627; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7628; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7629; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7630; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7631; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7632; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7633; GFX11-CU-NEXT:    s_endpgm
7634    i32* %out, i32 %in) {
7635entry:
7636  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") acquire
7637  ret void
7638}
7639
7640define amdgpu_kernel void @flat_singlethread_one_as_release_atomicrmw(
7641; GFX7-LABEL: flat_singlethread_one_as_release_atomicrmw:
7642; GFX7:       ; %bb.0: ; %entry
7643; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7644; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7645; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7646; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7647; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7648; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7649; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7650; GFX7-NEXT:    s_endpgm
7651;
7652; GFX10-WGP-LABEL: flat_singlethread_one_as_release_atomicrmw:
7653; GFX10-WGP:       ; %bb.0: ; %entry
7654; GFX10-WGP-NEXT:    s_clause 0x1
7655; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7656; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7657; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7658; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7659; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7660; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7661; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7662; GFX10-WGP-NEXT:    s_endpgm
7663;
7664; GFX10-CU-LABEL: flat_singlethread_one_as_release_atomicrmw:
7665; GFX10-CU:       ; %bb.0: ; %entry
7666; GFX10-CU-NEXT:    s_clause 0x1
7667; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7668; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7669; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7670; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7671; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7672; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7673; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7674; GFX10-CU-NEXT:    s_endpgm
7675;
7676; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_atomicrmw:
7677; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7678; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7679; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7680; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7681; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7682; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7683; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7684; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7685; SKIP-CACHE-INV-NEXT:    s_endpgm
7686;
7687; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_atomicrmw:
7688; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7689; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7690; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7691; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7692; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7693; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7694; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7695; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7696;
7697; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_atomicrmw:
7698; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7699; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7700; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7701; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7702; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7703; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7704; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7705; GFX90A-TGSPLIT-NEXT:    s_endpgm
7706;
7707; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_atomicrmw:
7708; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7709; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7710; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7711; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7712; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7713; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7714; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7715; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7716;
7717; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_atomicrmw:
7718; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7719; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7720; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7721; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7722; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7723; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7724; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7725; GFX940-TGSPLIT-NEXT:    s_endpgm
7726;
7727; GFX11-WGP-LABEL: flat_singlethread_one_as_release_atomicrmw:
7728; GFX11-WGP:       ; %bb.0: ; %entry
7729; GFX11-WGP-NEXT:    s_clause 0x1
7730; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7731; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
7732; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7733; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7734; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7735; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7736; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7737; GFX11-WGP-NEXT:    s_endpgm
7738;
7739; GFX11-CU-LABEL: flat_singlethread_one_as_release_atomicrmw:
7740; GFX11-CU:       ; %bb.0: ; %entry
7741; GFX11-CU-NEXT:    s_clause 0x1
7742; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7743; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7744; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7745; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7746; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7747; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7748; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7749; GFX11-CU-NEXT:    s_endpgm
7750    i32* %out, i32 %in) {
7751entry:
7752  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") release
7753  ret void
7754}
7755
7756define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_atomicrmw(
7757; GFX7-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
7758; GFX7:       ; %bb.0: ; %entry
7759; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7760; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7761; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7762; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7763; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7764; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7765; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7766; GFX7-NEXT:    s_endpgm
7767;
7768; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
7769; GFX10-WGP:       ; %bb.0: ; %entry
7770; GFX10-WGP-NEXT:    s_clause 0x1
7771; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7772; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7773; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7774; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7775; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7776; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7777; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7778; GFX10-WGP-NEXT:    s_endpgm
7779;
7780; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
7781; GFX10-CU:       ; %bb.0: ; %entry
7782; GFX10-CU-NEXT:    s_clause 0x1
7783; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7784; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7785; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7786; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7787; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7788; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7789; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7790; GFX10-CU-NEXT:    s_endpgm
7791;
7792; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
7793; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7794; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7795; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7796; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7797; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7798; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7799; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7800; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7801; SKIP-CACHE-INV-NEXT:    s_endpgm
7802;
7803; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
7804; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7805; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7806; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7807; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7808; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7809; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7810; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7811; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7812;
7813; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
7814; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7815; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7816; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7817; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7818; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7819; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7820; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7821; GFX90A-TGSPLIT-NEXT:    s_endpgm
7822;
7823; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
7824; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7825; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7826; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7827; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7828; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7829; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7830; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7831; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7832;
7833; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
7834; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7835; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7836; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7837; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7838; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7839; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7840; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7841; GFX940-TGSPLIT-NEXT:    s_endpgm
7842;
7843; GFX11-WGP-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
7844; GFX11-WGP:       ; %bb.0: ; %entry
7845; GFX11-WGP-NEXT:    s_clause 0x1
7846; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7847; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
7848; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7849; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7850; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7851; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7852; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7853; GFX11-WGP-NEXT:    s_endpgm
7854;
7855; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
7856; GFX11-CU:       ; %bb.0: ; %entry
7857; GFX11-CU-NEXT:    s_clause 0x1
7858; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7859; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7860; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7861; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7862; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7863; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7864; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7865; GFX11-CU-NEXT:    s_endpgm
7866    i32* %out, i32 %in) {
7867entry:
7868  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") acq_rel
7869  ret void
7870}
7871
7872define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_atomicrmw(
7873; GFX7-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
7874; GFX7:       ; %bb.0: ; %entry
7875; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7876; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7877; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7878; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7879; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7880; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7881; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7882; GFX7-NEXT:    s_endpgm
7883;
7884; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
7885; GFX10-WGP:       ; %bb.0: ; %entry
7886; GFX10-WGP-NEXT:    s_clause 0x1
7887; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7888; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7889; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7890; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7891; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7892; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7893; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7894; GFX10-WGP-NEXT:    s_endpgm
7895;
7896; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
7897; GFX10-CU:       ; %bb.0: ; %entry
7898; GFX10-CU-NEXT:    s_clause 0x1
7899; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7900; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7901; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7902; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7903; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7904; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7905; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7906; GFX10-CU-NEXT:    s_endpgm
7907;
7908; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
7909; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7910; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7911; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7912; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7913; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7914; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7915; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7916; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7917; SKIP-CACHE-INV-NEXT:    s_endpgm
7918;
7919; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
7920; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7921; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7922; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7923; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7924; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7925; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7926; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7927; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7928;
7929; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
7930; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7931; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7932; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7933; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7934; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7935; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7936; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7937; GFX90A-TGSPLIT-NEXT:    s_endpgm
7938;
7939; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
7940; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7941; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7942; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7943; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7944; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7945; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7946; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7947; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7948;
7949; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
7950; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7951; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7952; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7953; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7954; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7955; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7956; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7957; GFX940-TGSPLIT-NEXT:    s_endpgm
7958;
7959; GFX11-WGP-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
7960; GFX11-WGP:       ; %bb.0: ; %entry
7961; GFX11-WGP-NEXT:    s_clause 0x1
7962; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7963; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
7964; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7965; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7966; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7967; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7968; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7969; GFX11-WGP-NEXT:    s_endpgm
7970;
7971; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
7972; GFX11-CU:       ; %bb.0: ; %entry
7973; GFX11-CU-NEXT:    s_clause 0x1
7974; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7975; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
7976; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7977; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7978; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7979; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7980; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
7981; GFX11-CU-NEXT:    s_endpgm
7982    i32* %out, i32 %in) {
7983entry:
7984  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") seq_cst
7985  ret void
7986}
7987
7988define amdgpu_kernel void @flat_singlethread_one_as_acquire_ret_atomicrmw(
7989; GFX7-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
7990; GFX7:       ; %bb.0: ; %entry
7991; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7992; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7993; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7994; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7995; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7996; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7997; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7998; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7999; GFX7-NEXT:    flat_store_dword v[0:1], v2
8000; GFX7-NEXT:    s_endpgm
8001;
8002; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
8003; GFX10-WGP:       ; %bb.0: ; %entry
8004; GFX10-WGP-NEXT:    s_clause 0x1
8005; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8006; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8007; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8008; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8009; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8010; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8011; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8012; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8013; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8014; GFX10-WGP-NEXT:    s_endpgm
8015;
8016; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
8017; GFX10-CU:       ; %bb.0: ; %entry
8018; GFX10-CU-NEXT:    s_clause 0x1
8019; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8020; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8021; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8022; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8023; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8024; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8025; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8026; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8027; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8028; GFX10-CU-NEXT:    s_endpgm
8029;
8030; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
8031; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8032; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8033; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8034; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8035; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8036; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8037; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8038; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8039; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8040; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8041; SKIP-CACHE-INV-NEXT:    s_endpgm
8042;
8043; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
8044; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8045; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8046; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8047; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8048; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8049; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8050; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8051; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8052; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8053; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8054;
8055; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
8056; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8057; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8058; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8059; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8060; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8061; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8062; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8063; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8064; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8065; GFX90A-TGSPLIT-NEXT:    s_endpgm
8066;
8067; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
8068; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8069; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8070; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8071; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8072; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8073; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8074; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8075; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8076; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8077; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8078;
8079; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
8080; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8081; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8082; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8083; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8084; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8085; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8086; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8087; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8088; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8089; GFX940-TGSPLIT-NEXT:    s_endpgm
8090;
8091; GFX11-WGP-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
8092; GFX11-WGP:       ; %bb.0: ; %entry
8093; GFX11-WGP-NEXT:    s_clause 0x1
8094; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8095; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8096; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8097; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8098; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8099; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8100; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8101; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8102; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8103; GFX11-WGP-NEXT:    s_endpgm
8104;
8105; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
8106; GFX11-CU:       ; %bb.0: ; %entry
8107; GFX11-CU-NEXT:    s_clause 0x1
8108; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8109; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8110; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8111; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8112; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8113; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8114; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8115; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8116; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8117; GFX11-CU-NEXT:    s_endpgm
8118    i32* %out, i32 %in) {
8119entry:
8120  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") acquire
8121  store i32 %val, i32* %out, align 4
8122  ret void
8123}
8124
8125define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_ret_atomicrmw(
8126; GFX7-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
8127; GFX7:       ; %bb.0: ; %entry
8128; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8129; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8130; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8131; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8132; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8133; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8134; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8135; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8136; GFX7-NEXT:    flat_store_dword v[0:1], v2
8137; GFX7-NEXT:    s_endpgm
8138;
8139; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
8140; GFX10-WGP:       ; %bb.0: ; %entry
8141; GFX10-WGP-NEXT:    s_clause 0x1
8142; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8143; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8144; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8145; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8146; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8147; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8148; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8149; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8150; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8151; GFX10-WGP-NEXT:    s_endpgm
8152;
8153; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
8154; GFX10-CU:       ; %bb.0: ; %entry
8155; GFX10-CU-NEXT:    s_clause 0x1
8156; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8157; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8158; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8159; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8160; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8161; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8162; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8163; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8164; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8165; GFX10-CU-NEXT:    s_endpgm
8166;
8167; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
8168; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8169; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8170; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8171; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8172; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8173; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8174; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8175; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8176; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8177; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8178; SKIP-CACHE-INV-NEXT:    s_endpgm
8179;
8180; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
8181; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8182; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8183; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8184; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8185; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8186; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8187; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8188; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8189; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8190; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8191;
8192; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
8193; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8194; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8195; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8196; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8197; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8198; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8199; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8200; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8201; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8202; GFX90A-TGSPLIT-NEXT:    s_endpgm
8203;
8204; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
8205; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8206; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8207; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8208; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8209; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8210; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8211; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8212; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8213; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8214; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8215;
8216; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
8217; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8218; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8219; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8220; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8221; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8222; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8223; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8224; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8225; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8226; GFX940-TGSPLIT-NEXT:    s_endpgm
8227;
8228; GFX11-WGP-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
8229; GFX11-WGP:       ; %bb.0: ; %entry
8230; GFX11-WGP-NEXT:    s_clause 0x1
8231; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8232; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8233; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8234; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8235; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8236; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8237; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8238; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8239; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8240; GFX11-WGP-NEXT:    s_endpgm
8241;
8242; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
8243; GFX11-CU:       ; %bb.0: ; %entry
8244; GFX11-CU-NEXT:    s_clause 0x1
8245; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8246; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8247; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8248; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8249; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8250; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8251; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8252; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8253; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8254; GFX11-CU-NEXT:    s_endpgm
8255    i32* %out, i32 %in) {
8256entry:
8257  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") acq_rel
8258  store i32 %val, i32* %out, align 4
8259  ret void
8260}
8261
8262define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_ret_atomicrmw(
8263; GFX7-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
8264; GFX7:       ; %bb.0: ; %entry
8265; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8266; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8267; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8268; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8269; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8270; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8271; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8272; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8273; GFX7-NEXT:    flat_store_dword v[0:1], v2
8274; GFX7-NEXT:    s_endpgm
8275;
8276; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
8277; GFX10-WGP:       ; %bb.0: ; %entry
8278; GFX10-WGP-NEXT:    s_clause 0x1
8279; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8280; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8281; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8282; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8283; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8284; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8285; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8286; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8287; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8288; GFX10-WGP-NEXT:    s_endpgm
8289;
8290; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
8291; GFX10-CU:       ; %bb.0: ; %entry
8292; GFX10-CU-NEXT:    s_clause 0x1
8293; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8294; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8295; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8296; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8297; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8298; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8299; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8300; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8301; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8302; GFX10-CU-NEXT:    s_endpgm
8303;
8304; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
8305; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8306; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8307; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8308; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8309; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8310; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8311; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8312; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8313; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8314; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8315; SKIP-CACHE-INV-NEXT:    s_endpgm
8316;
8317; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
8318; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8319; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8320; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8321; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8322; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8323; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8324; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8325; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8326; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8327; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8328;
8329; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
8330; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8331; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8332; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8333; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8334; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8335; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8336; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8337; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8338; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8339; GFX90A-TGSPLIT-NEXT:    s_endpgm
8340;
8341; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
8342; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8343; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8344; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8345; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8346; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8347; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8348; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8349; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8350; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8351; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8352;
8353; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
8354; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8355; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8356; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8357; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8358; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8359; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8360; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
8361; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8362; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8363; GFX940-TGSPLIT-NEXT:    s_endpgm
8364;
8365; GFX11-WGP-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
8366; GFX11-WGP:       ; %bb.0: ; %entry
8367; GFX11-WGP-NEXT:    s_clause 0x1
8368; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8369; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8370; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8371; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8372; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8373; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8374; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8375; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8376; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8377; GFX11-WGP-NEXT:    s_endpgm
8378;
8379; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
8380; GFX11-CU:       ; %bb.0: ; %entry
8381; GFX11-CU-NEXT:    s_clause 0x1
8382; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8383; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8384; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8385; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8386; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8387; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8388; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
8389; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8390; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8391; GFX11-CU-NEXT:    s_endpgm
8392    i32* %out, i32 %in) {
8393entry:
8394  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") seq_cst
8395  store i32 %val, i32* %out, align 4
8396  ret void
8397}
8398
8399define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_cmpxchg(
8400; GFX7-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
8401; GFX7:       ; %bb.0: ; %entry
8402; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8403; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8404; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8405; GFX7-NEXT:    s_add_u32 s0, s0, 16
8406; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8407; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8408; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8409; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8410; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8411; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8412; GFX7-NEXT:    s_endpgm
8413;
8414; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
8415; GFX10-WGP:       ; %bb.0: ; %entry
8416; GFX10-WGP-NEXT:    s_clause 0x1
8417; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8418; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8419; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8420; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8421; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8422; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8423; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8424; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8425; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8426; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8427; GFX10-WGP-NEXT:    s_endpgm
8428;
8429; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
8430; GFX10-CU:       ; %bb.0: ; %entry
8431; GFX10-CU-NEXT:    s_clause 0x1
8432; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8433; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8434; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8435; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8436; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8437; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8438; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8439; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8440; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8441; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8442; GFX10-CU-NEXT:    s_endpgm
8443;
8444; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
8445; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8446; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8447; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8448; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8449; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8450; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8451; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8452; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8453; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8454; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8455; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8456; SKIP-CACHE-INV-NEXT:    s_endpgm
8457;
8458; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
8459; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8460; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8461; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8462; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8463; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8464; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8465; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8466; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8467;
8468; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
8469; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8470; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8471; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8472; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8473; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8474; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8475; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8476; GFX90A-TGSPLIT-NEXT:    s_endpgm
8477;
8478; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
8479; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8480; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8481; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8482; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8483; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8484; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8485; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8486; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8487;
8488; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
8489; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8490; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8491; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8492; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8493; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8494; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8495; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8496; GFX940-TGSPLIT-NEXT:    s_endpgm
8497;
8498; GFX11-WGP-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
8499; GFX11-WGP:       ; %bb.0: ; %entry
8500; GFX11-WGP-NEXT:    s_clause 0x1
8501; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8502; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8503; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8504; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8505; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
8506; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8507; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8508; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8509; GFX11-WGP-NEXT:    s_endpgm
8510;
8511; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
8512; GFX11-CU:       ; %bb.0: ; %entry
8513; GFX11-CU-NEXT:    s_clause 0x1
8514; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8515; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8516; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8517; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8518; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
8519; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8520; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8521; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8522; GFX11-CU-NEXT:    s_endpgm
8523    i32* %out, i32 %in, i32 %old) {
8524entry:
8525  %gep = getelementptr i32, i32* %out, i32 4
8526  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
8527  ret void
8528}
8529
8530define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_cmpxchg(
8531; GFX7-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
8532; GFX7:       ; %bb.0: ; %entry
8533; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8534; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8535; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8536; GFX7-NEXT:    s_add_u32 s0, s0, 16
8537; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8538; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8539; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8540; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8541; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8542; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8543; GFX7-NEXT:    s_endpgm
8544;
8545; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
8546; GFX10-WGP:       ; %bb.0: ; %entry
8547; GFX10-WGP-NEXT:    s_clause 0x1
8548; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8549; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8550; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8551; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8552; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8553; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8554; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8555; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8556; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8557; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8558; GFX10-WGP-NEXT:    s_endpgm
8559;
8560; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
8561; GFX10-CU:       ; %bb.0: ; %entry
8562; GFX10-CU-NEXT:    s_clause 0x1
8563; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8564; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8565; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8566; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8567; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8568; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8569; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8570; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8571; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8572; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8573; GFX10-CU-NEXT:    s_endpgm
8574;
8575; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
8576; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8577; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8578; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8579; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8580; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8581; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8582; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8583; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8584; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8585; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8586; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8587; SKIP-CACHE-INV-NEXT:    s_endpgm
8588;
8589; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
8590; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8591; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8592; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8593; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8594; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8595; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8596; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8597; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8598;
8599; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
8600; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8601; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8602; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8603; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8604; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8605; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8606; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8607; GFX90A-TGSPLIT-NEXT:    s_endpgm
8608;
8609; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
8610; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8611; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8612; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8613; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8614; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8615; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8616; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8617; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8618;
8619; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
8620; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8621; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8622; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8623; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8624; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8625; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8626; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8627; GFX940-TGSPLIT-NEXT:    s_endpgm
8628;
8629; GFX11-WGP-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
8630; GFX11-WGP:       ; %bb.0: ; %entry
8631; GFX11-WGP-NEXT:    s_clause 0x1
8632; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8633; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8634; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8635; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8636; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
8637; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8638; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8639; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8640; GFX11-WGP-NEXT:    s_endpgm
8641;
8642; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
8643; GFX11-CU:       ; %bb.0: ; %entry
8644; GFX11-CU-NEXT:    s_clause 0x1
8645; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8646; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8647; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8648; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8649; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
8650; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8651; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8652; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8653; GFX11-CU-NEXT:    s_endpgm
8654    i32* %out, i32 %in, i32 %old) {
8655entry:
8656  %gep = getelementptr i32, i32* %out, i32 4
8657  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
8658  ret void
8659}
8660
8661define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_cmpxchg(
8662; GFX7-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
8663; GFX7:       ; %bb.0: ; %entry
8664; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8665; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8666; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8667; GFX7-NEXT:    s_add_u32 s0, s0, 16
8668; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8669; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8670; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8671; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8672; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8673; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8674; GFX7-NEXT:    s_endpgm
8675;
8676; GFX10-WGP-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
8677; GFX10-WGP:       ; %bb.0: ; %entry
8678; GFX10-WGP-NEXT:    s_clause 0x1
8679; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8680; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8681; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8682; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8683; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8684; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8685; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8686; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8687; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8688; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8689; GFX10-WGP-NEXT:    s_endpgm
8690;
8691; GFX10-CU-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
8692; GFX10-CU:       ; %bb.0: ; %entry
8693; GFX10-CU-NEXT:    s_clause 0x1
8694; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8695; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8696; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8697; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8698; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8699; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8700; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8701; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8702; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8703; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8704; GFX10-CU-NEXT:    s_endpgm
8705;
8706; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
8707; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8708; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8709; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8710; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8711; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8712; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8713; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8714; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8715; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8716; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8717; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8718; SKIP-CACHE-INV-NEXT:    s_endpgm
8719;
8720; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
8721; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8722; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8723; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8724; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8725; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8726; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8727; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8728; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8729;
8730; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
8731; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8732; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8733; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8734; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8735; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8736; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8737; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8738; GFX90A-TGSPLIT-NEXT:    s_endpgm
8739;
8740; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
8741; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8742; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8743; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8744; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8745; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8746; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8747; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8748; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8749;
8750; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
8751; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8752; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8753; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8754; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8755; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8756; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8757; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8758; GFX940-TGSPLIT-NEXT:    s_endpgm
8759;
8760; GFX11-WGP-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
8761; GFX11-WGP:       ; %bb.0: ; %entry
8762; GFX11-WGP-NEXT:    s_clause 0x1
8763; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8764; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8765; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8766; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8767; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
8768; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8769; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8770; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8771; GFX11-WGP-NEXT:    s_endpgm
8772;
8773; GFX11-CU-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
8774; GFX11-CU:       ; %bb.0: ; %entry
8775; GFX11-CU-NEXT:    s_clause 0x1
8776; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8777; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8778; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8779; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8780; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
8781; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8782; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8783; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8784; GFX11-CU-NEXT:    s_endpgm
8785    i32* %out, i32 %in, i32 %old) {
8786entry:
8787  %gep = getelementptr i32, i32* %out, i32 4
8788  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
8789  ret void
8790}
8791
8792define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_cmpxchg(
8793; GFX7-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8794; GFX7:       ; %bb.0: ; %entry
8795; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8796; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8797; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8798; GFX7-NEXT:    s_add_u32 s0, s0, 16
8799; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8800; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8801; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8802; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8803; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8804; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8805; GFX7-NEXT:    s_endpgm
8806;
8807; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8808; GFX10-WGP:       ; %bb.0: ; %entry
8809; GFX10-WGP-NEXT:    s_clause 0x1
8810; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8811; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8812; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8813; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8814; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8815; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8816; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8817; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8818; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8819; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8820; GFX10-WGP-NEXT:    s_endpgm
8821;
8822; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8823; GFX10-CU:       ; %bb.0: ; %entry
8824; GFX10-CU-NEXT:    s_clause 0x1
8825; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8826; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8827; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8828; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8829; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8830; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8831; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8832; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8833; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8834; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8835; GFX10-CU-NEXT:    s_endpgm
8836;
8837; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8838; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8839; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8840; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8841; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8842; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8843; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8844; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8845; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8846; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8847; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8848; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8849; SKIP-CACHE-INV-NEXT:    s_endpgm
8850;
8851; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8852; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8853; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8854; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8855; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8856; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8857; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8858; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8859; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8860;
8861; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8862; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8863; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8864; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8865; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8866; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8867; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8868; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8869; GFX90A-TGSPLIT-NEXT:    s_endpgm
8870;
8871; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8872; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8873; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8874; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8875; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8876; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8877; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8878; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8879; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8880;
8881; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8882; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8883; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8884; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8885; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8886; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8887; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8888; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8889; GFX940-TGSPLIT-NEXT:    s_endpgm
8890;
8891; GFX11-WGP-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8892; GFX11-WGP:       ; %bb.0: ; %entry
8893; GFX11-WGP-NEXT:    s_clause 0x1
8894; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8895; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8896; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8897; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8898; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
8899; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8900; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8901; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8902; GFX11-WGP-NEXT:    s_endpgm
8903;
8904; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8905; GFX11-CU:       ; %bb.0: ; %entry
8906; GFX11-CU-NEXT:    s_clause 0x1
8907; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8908; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
8909; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8910; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8911; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
8912; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8913; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8914; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8915; GFX11-CU-NEXT:    s_endpgm
8916    i32* %out, i32 %in, i32 %old) {
8917entry:
8918  %gep = getelementptr i32, i32* %out, i32 4
8919  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
8920  ret void
8921}
8922
8923define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_cmpxchg(
8924; GFX7-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8925; GFX7:       ; %bb.0: ; %entry
8926; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8927; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8928; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8929; GFX7-NEXT:    s_add_u32 s0, s0, 16
8930; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8931; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8932; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8933; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8934; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8935; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8936; GFX7-NEXT:    s_endpgm
8937;
8938; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8939; GFX10-WGP:       ; %bb.0: ; %entry
8940; GFX10-WGP-NEXT:    s_clause 0x1
8941; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8942; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8943; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8944; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8945; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8946; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8947; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8948; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8949; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8950; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8951; GFX10-WGP-NEXT:    s_endpgm
8952;
8953; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8954; GFX10-CU:       ; %bb.0: ; %entry
8955; GFX10-CU-NEXT:    s_clause 0x1
8956; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8957; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8958; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8959; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8960; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8961; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8962; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8963; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8964; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8965; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8966; GFX10-CU-NEXT:    s_endpgm
8967;
8968; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8969; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8970; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8971; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8972; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8973; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8974; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8975; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8976; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8977; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8978; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8979; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8980; SKIP-CACHE-INV-NEXT:    s_endpgm
8981;
8982; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8983; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8984; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8985; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8986; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8987; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8988; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8989; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8990; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8991;
8992; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8993; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8994; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8995; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8996; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8997; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8998; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8999; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9000; GFX90A-TGSPLIT-NEXT:    s_endpgm
9001;
9002; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9003; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9004; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9005; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9006; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9007; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9008; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9009; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9010; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9011;
9012; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9013; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9014; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9015; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9016; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9017; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9018; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9019; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9020; GFX940-TGSPLIT-NEXT:    s_endpgm
9021;
9022; GFX11-WGP-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9023; GFX11-WGP:       ; %bb.0: ; %entry
9024; GFX11-WGP-NEXT:    s_clause 0x1
9025; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9026; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9027; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9028; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9029; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9030; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9031; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9032; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9033; GFX11-WGP-NEXT:    s_endpgm
9034;
9035; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9036; GFX11-CU:       ; %bb.0: ; %entry
9037; GFX11-CU-NEXT:    s_clause 0x1
9038; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9039; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9040; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9041; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9042; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9043; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9044; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9045; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9046; GFX11-CU-NEXT:    s_endpgm
9047    i32* %out, i32 %in, i32 %old) {
9048entry:
9049  %gep = getelementptr i32, i32* %out, i32 4
9050  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
9051  ret void
9052}
9053
9054define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_cmpxchg(
9055; GFX7-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
9056; GFX7:       ; %bb.0: ; %entry
9057; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9058; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9059; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9060; GFX7-NEXT:    s_add_u32 s0, s0, 16
9061; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9062; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9063; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9064; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9065; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9066; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9067; GFX7-NEXT:    s_endpgm
9068;
9069; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
9070; GFX10-WGP:       ; %bb.0: ; %entry
9071; GFX10-WGP-NEXT:    s_clause 0x1
9072; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9073; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9074; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9075; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9076; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9077; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9078; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9079; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9080; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9081; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9082; GFX10-WGP-NEXT:    s_endpgm
9083;
9084; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
9085; GFX10-CU:       ; %bb.0: ; %entry
9086; GFX10-CU-NEXT:    s_clause 0x1
9087; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9088; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9089; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9090; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9091; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9092; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9093; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9094; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9095; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9096; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9097; GFX10-CU-NEXT:    s_endpgm
9098;
9099; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
9100; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9101; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9102; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9103; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9104; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9105; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9106; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9107; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9108; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9109; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9110; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9111; SKIP-CACHE-INV-NEXT:    s_endpgm
9112;
9113; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
9114; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9115; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9116; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9117; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9118; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9119; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9120; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9121; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9122;
9123; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
9124; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9125; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9126; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9127; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9128; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9129; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9130; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9131; GFX90A-TGSPLIT-NEXT:    s_endpgm
9132;
9133; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
9134; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9135; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9136; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9137; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9138; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9139; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9140; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9141; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9142;
9143; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
9144; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9145; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9146; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9147; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9148; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9149; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9150; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9151; GFX940-TGSPLIT-NEXT:    s_endpgm
9152;
9153; GFX11-WGP-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
9154; GFX11-WGP:       ; %bb.0: ; %entry
9155; GFX11-WGP-NEXT:    s_clause 0x1
9156; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9157; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9158; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9159; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9160; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9161; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9162; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9163; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9164; GFX11-WGP-NEXT:    s_endpgm
9165;
9166; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
9167; GFX11-CU:       ; %bb.0: ; %entry
9168; GFX11-CU-NEXT:    s_clause 0x1
9169; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9170; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9171; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9172; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9173; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9174; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9175; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9176; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9177; GFX11-CU-NEXT:    s_endpgm
9178    i32* %out, i32 %in, i32 %old) {
9179entry:
9180  %gep = getelementptr i32, i32* %out, i32 4
9181  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic acquire
9182  ret void
9183}
9184
9185define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_cmpxchg(
9186; GFX7-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
9187; GFX7:       ; %bb.0: ; %entry
9188; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9189; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9190; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9191; GFX7-NEXT:    s_add_u32 s0, s0, 16
9192; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9193; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9194; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9195; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9196; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9197; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9198; GFX7-NEXT:    s_endpgm
9199;
9200; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
9201; GFX10-WGP:       ; %bb.0: ; %entry
9202; GFX10-WGP-NEXT:    s_clause 0x1
9203; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9204; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9205; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9206; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9207; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9208; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9209; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9210; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9211; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9212; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9213; GFX10-WGP-NEXT:    s_endpgm
9214;
9215; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
9216; GFX10-CU:       ; %bb.0: ; %entry
9217; GFX10-CU-NEXT:    s_clause 0x1
9218; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9219; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9220; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9221; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9222; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9223; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9224; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9225; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9226; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9227; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9228; GFX10-CU-NEXT:    s_endpgm
9229;
9230; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
9231; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9232; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9233; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9234; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9235; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9236; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9237; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9238; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9239; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9240; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9241; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9242; SKIP-CACHE-INV-NEXT:    s_endpgm
9243;
9244; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
9245; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9246; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9247; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9248; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9249; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9250; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9251; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9252; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9253;
9254; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
9255; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9256; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9257; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9258; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9259; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9260; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9261; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9262; GFX90A-TGSPLIT-NEXT:    s_endpgm
9263;
9264; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
9265; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9266; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9267; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9268; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9269; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9270; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9271; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9272; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9273;
9274; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
9275; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9276; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9277; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9278; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9279; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9280; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9281; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9282; GFX940-TGSPLIT-NEXT:    s_endpgm
9283;
9284; GFX11-WGP-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
9285; GFX11-WGP:       ; %bb.0: ; %entry
9286; GFX11-WGP-NEXT:    s_clause 0x1
9287; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9288; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9289; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9290; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9291; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9292; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9293; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9294; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9295; GFX11-WGP-NEXT:    s_endpgm
9296;
9297; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
9298; GFX11-CU:       ; %bb.0: ; %entry
9299; GFX11-CU-NEXT:    s_clause 0x1
9300; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9301; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9302; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9303; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9304; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9305; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9306; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9307; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9308; GFX11-CU-NEXT:    s_endpgm
9309    i32* %out, i32 %in, i32 %old) {
9310entry:
9311  %gep = getelementptr i32, i32* %out, i32 4
9312  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
9313  ret void
9314}
9315
9316define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_cmpxchg(
9317; GFX7-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
9318; GFX7:       ; %bb.0: ; %entry
9319; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9320; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9321; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9322; GFX7-NEXT:    s_add_u32 s0, s0, 16
9323; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9324; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9325; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9326; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9327; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9328; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9329; GFX7-NEXT:    s_endpgm
9330;
9331; GFX10-WGP-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
9332; GFX10-WGP:       ; %bb.0: ; %entry
9333; GFX10-WGP-NEXT:    s_clause 0x1
9334; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9335; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9336; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9337; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9338; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9339; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9340; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9341; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9342; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9343; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9344; GFX10-WGP-NEXT:    s_endpgm
9345;
9346; GFX10-CU-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
9347; GFX10-CU:       ; %bb.0: ; %entry
9348; GFX10-CU-NEXT:    s_clause 0x1
9349; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9350; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9351; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9352; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9353; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9354; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9355; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9356; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9357; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9358; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9359; GFX10-CU-NEXT:    s_endpgm
9360;
9361; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
9362; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9363; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9364; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9365; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9366; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9367; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9368; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9369; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9370; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9371; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9372; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9373; SKIP-CACHE-INV-NEXT:    s_endpgm
9374;
9375; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
9376; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9377; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9378; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9379; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9380; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9381; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9382; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9383; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9384;
9385; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
9386; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9387; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9388; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9389; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9390; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9391; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9392; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9393; GFX90A-TGSPLIT-NEXT:    s_endpgm
9394;
9395; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
9396; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9397; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9398; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9399; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9400; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9401; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9402; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9403; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9404;
9405; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
9406; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9407; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9408; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9409; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9410; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9411; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9412; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9413; GFX940-TGSPLIT-NEXT:    s_endpgm
9414;
9415; GFX11-WGP-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
9416; GFX11-WGP:       ; %bb.0: ; %entry
9417; GFX11-WGP-NEXT:    s_clause 0x1
9418; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9419; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9420; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9421; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9422; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9423; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9424; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9425; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9426; GFX11-WGP-NEXT:    s_endpgm
9427;
9428; GFX11-CU-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
9429; GFX11-CU:       ; %bb.0: ; %entry
9430; GFX11-CU-NEXT:    s_clause 0x1
9431; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9432; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9433; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9434; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9435; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9436; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9437; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9438; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9439; GFX11-CU-NEXT:    s_endpgm
9440    i32* %out, i32 %in, i32 %old) {
9441entry:
9442  %gep = getelementptr i32, i32* %out, i32 4
9443  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
9444  ret void
9445}
9446
9447define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_cmpxchg(
9448; GFX7-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
9449; GFX7:       ; %bb.0: ; %entry
9450; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9451; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9452; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9453; GFX7-NEXT:    s_add_u32 s0, s0, 16
9454; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9455; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9456; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9457; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9458; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9459; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9460; GFX7-NEXT:    s_endpgm
9461;
9462; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
9463; GFX10-WGP:       ; %bb.0: ; %entry
9464; GFX10-WGP-NEXT:    s_clause 0x1
9465; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9466; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9467; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9468; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9469; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9470; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9471; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9472; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9473; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9474; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9475; GFX10-WGP-NEXT:    s_endpgm
9476;
9477; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
9478; GFX10-CU:       ; %bb.0: ; %entry
9479; GFX10-CU-NEXT:    s_clause 0x1
9480; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9481; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9482; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9483; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9484; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9485; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9486; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9487; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9488; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9489; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9490; GFX10-CU-NEXT:    s_endpgm
9491;
9492; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
9493; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9494; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9495; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9496; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9497; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9498; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9499; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9500; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9501; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9502; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9503; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9504; SKIP-CACHE-INV-NEXT:    s_endpgm
9505;
9506; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
9507; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9508; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9509; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9510; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9511; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9512; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9513; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9514; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9515;
9516; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
9517; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9518; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9519; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9520; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9521; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9522; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9523; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9524; GFX90A-TGSPLIT-NEXT:    s_endpgm
9525;
9526; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
9527; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9528; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9529; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9530; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9531; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9532; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9533; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9534; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9535;
9536; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
9537; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9538; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9539; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9540; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9541; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9542; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9543; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9544; GFX940-TGSPLIT-NEXT:    s_endpgm
9545;
9546; GFX11-WGP-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
9547; GFX11-WGP:       ; %bb.0: ; %entry
9548; GFX11-WGP-NEXT:    s_clause 0x1
9549; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9550; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9551; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9552; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9553; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9554; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9555; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9556; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9557; GFX11-WGP-NEXT:    s_endpgm
9558;
9559; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
9560; GFX11-CU:       ; %bb.0: ; %entry
9561; GFX11-CU-NEXT:    s_clause 0x1
9562; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9563; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9564; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9565; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9566; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9567; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9568; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9569; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9570; GFX11-CU-NEXT:    s_endpgm
9571    i32* %out, i32 %in, i32 %old) {
9572entry:
9573  %gep = getelementptr i32, i32* %out, i32 4
9574  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
9575  ret void
9576}
9577
9578define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_cmpxchg(
9579; GFX7-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
9580; GFX7:       ; %bb.0: ; %entry
9581; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9582; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9583; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9584; GFX7-NEXT:    s_add_u32 s0, s0, 16
9585; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9586; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9587; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9588; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9589; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9590; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9591; GFX7-NEXT:    s_endpgm
9592;
9593; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
9594; GFX10-WGP:       ; %bb.0: ; %entry
9595; GFX10-WGP-NEXT:    s_clause 0x1
9596; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9597; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9598; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9599; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9600; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9601; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9602; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9603; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9604; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9605; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9606; GFX10-WGP-NEXT:    s_endpgm
9607;
9608; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
9609; GFX10-CU:       ; %bb.0: ; %entry
9610; GFX10-CU-NEXT:    s_clause 0x1
9611; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9612; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9613; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9614; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9615; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9616; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9617; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9618; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9619; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9620; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9621; GFX10-CU-NEXT:    s_endpgm
9622;
9623; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
9624; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9625; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9626; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9627; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9628; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9629; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9630; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9631; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9632; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9633; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9634; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9635; SKIP-CACHE-INV-NEXT:    s_endpgm
9636;
9637; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
9638; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9639; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9640; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9641; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9642; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9643; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9644; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9645; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9646;
9647; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
9648; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9649; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9650; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9651; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9652; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9653; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9654; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9655; GFX90A-TGSPLIT-NEXT:    s_endpgm
9656;
9657; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
9658; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9659; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9660; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9661; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9662; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9663; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9664; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9665; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9666;
9667; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
9668; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9669; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9670; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9671; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9672; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9673; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9674; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9675; GFX940-TGSPLIT-NEXT:    s_endpgm
9676;
9677; GFX11-WGP-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
9678; GFX11-WGP:       ; %bb.0: ; %entry
9679; GFX11-WGP-NEXT:    s_clause 0x1
9680; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9681; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9682; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9683; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9684; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9685; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9686; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9687; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9688; GFX11-WGP-NEXT:    s_endpgm
9689;
9690; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
9691; GFX11-CU:       ; %bb.0: ; %entry
9692; GFX11-CU-NEXT:    s_clause 0x1
9693; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9694; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9695; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9696; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9697; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9698; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9699; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9700; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9701; GFX11-CU-NEXT:    s_endpgm
9702    i32* %out, i32 %in, i32 %old) {
9703entry:
9704  %gep = getelementptr i32, i32* %out, i32 4
9705  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
9706  ret void
9707}
9708
9709define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_cmpxchg(
9710; GFX7-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9711; GFX7:       ; %bb.0: ; %entry
9712; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9713; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9714; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9715; GFX7-NEXT:    s_add_u32 s0, s0, 16
9716; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9717; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9718; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9719; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9720; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9721; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9722; GFX7-NEXT:    s_endpgm
9723;
9724; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9725; GFX10-WGP:       ; %bb.0: ; %entry
9726; GFX10-WGP-NEXT:    s_clause 0x1
9727; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9728; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9729; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9730; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9731; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9732; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9733; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9734; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9735; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9736; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9737; GFX10-WGP-NEXT:    s_endpgm
9738;
9739; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9740; GFX10-CU:       ; %bb.0: ; %entry
9741; GFX10-CU-NEXT:    s_clause 0x1
9742; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9743; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9744; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9745; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9746; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9747; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9748; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9749; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9750; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9751; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9752; GFX10-CU-NEXT:    s_endpgm
9753;
9754; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9755; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9756; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9757; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9758; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9759; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9760; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9761; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9762; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9763; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9764; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9765; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9766; SKIP-CACHE-INV-NEXT:    s_endpgm
9767;
9768; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9769; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9770; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9771; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9772; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9773; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9774; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9775; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9776; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9777;
9778; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9779; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9780; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9781; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9782; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9783; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9784; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9785; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9786; GFX90A-TGSPLIT-NEXT:    s_endpgm
9787;
9788; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9789; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9790; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9791; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9792; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9793; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9794; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9795; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9796; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9797;
9798; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9799; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9800; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9801; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9802; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9803; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9804; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9805; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9806; GFX940-TGSPLIT-NEXT:    s_endpgm
9807;
9808; GFX11-WGP-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9809; GFX11-WGP:       ; %bb.0: ; %entry
9810; GFX11-WGP-NEXT:    s_clause 0x1
9811; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9812; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9813; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9814; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9815; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9816; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9817; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9818; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9819; GFX11-WGP-NEXT:    s_endpgm
9820;
9821; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9822; GFX11-CU:       ; %bb.0: ; %entry
9823; GFX11-CU-NEXT:    s_clause 0x1
9824; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9825; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9826; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9827; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9828; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9829; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9830; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9831; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9832; GFX11-CU-NEXT:    s_endpgm
9833    i32* %out, i32 %in, i32 %old) {
9834entry:
9835  %gep = getelementptr i32, i32* %out, i32 4
9836  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic seq_cst
9837  ret void
9838}
9839
9840define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_cmpxchg(
9841; GFX7-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
9842; GFX7:       ; %bb.0: ; %entry
9843; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9844; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9845; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9846; GFX7-NEXT:    s_add_u32 s0, s0, 16
9847; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9848; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9849; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9850; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9851; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9852; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9853; GFX7-NEXT:    s_endpgm
9854;
9855; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
9856; GFX10-WGP:       ; %bb.0: ; %entry
9857; GFX10-WGP-NEXT:    s_clause 0x1
9858; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9859; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9860; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9861; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9862; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9863; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9864; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9865; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9866; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9867; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9868; GFX10-WGP-NEXT:    s_endpgm
9869;
9870; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
9871; GFX10-CU:       ; %bb.0: ; %entry
9872; GFX10-CU-NEXT:    s_clause 0x1
9873; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9874; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9875; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9876; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9877; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9878; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9879; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9880; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9881; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9882; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9883; GFX10-CU-NEXT:    s_endpgm
9884;
9885; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
9886; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9887; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9888; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9889; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9890; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9891; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9892; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9893; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9894; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9895; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9896; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9897; SKIP-CACHE-INV-NEXT:    s_endpgm
9898;
9899; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
9900; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9901; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9902; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9903; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9904; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9905; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9906; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9907; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9908;
9909; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
9910; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9911; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9912; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9913; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9914; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9915; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9916; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9917; GFX90A-TGSPLIT-NEXT:    s_endpgm
9918;
9919; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
9920; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9921; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9922; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9923; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9924; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9925; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9926; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9927; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9928;
9929; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
9930; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9931; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9932; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9933; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9934; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9935; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9936; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9937; GFX940-TGSPLIT-NEXT:    s_endpgm
9938;
9939; GFX11-WGP-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
9940; GFX11-WGP:       ; %bb.0: ; %entry
9941; GFX11-WGP-NEXT:    s_clause 0x1
9942; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9943; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9944; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9945; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9946; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9947; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9948; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9949; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9950; GFX11-WGP-NEXT:    s_endpgm
9951;
9952; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
9953; GFX11-CU:       ; %bb.0: ; %entry
9954; GFX11-CU-NEXT:    s_clause 0x1
9955; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9956; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9957; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9958; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9959; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9960; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9961; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9962; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9963; GFX11-CU-NEXT:    s_endpgm
9964    i32* %out, i32 %in, i32 %old) {
9965entry:
9966  %gep = getelementptr i32, i32* %out, i32 4
9967  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire seq_cst
9968  ret void
9969}
9970
9971define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_cmpxchg(
9972; GFX7-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
9973; GFX7:       ; %bb.0: ; %entry
9974; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9975; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9976; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9977; GFX7-NEXT:    s_add_u32 s0, s0, 16
9978; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9979; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9980; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9981; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9982; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9983; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9984; GFX7-NEXT:    s_endpgm
9985;
9986; GFX10-WGP-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
9987; GFX10-WGP:       ; %bb.0: ; %entry
9988; GFX10-WGP-NEXT:    s_clause 0x1
9989; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9990; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9991; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9992; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9993; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9994; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9995; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9996; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9997; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9998; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9999; GFX10-WGP-NEXT:    s_endpgm
10000;
10001; GFX10-CU-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
10002; GFX10-CU:       ; %bb.0: ; %entry
10003; GFX10-CU-NEXT:    s_clause 0x1
10004; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10005; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10006; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10007; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10008; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10009; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10010; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10011; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10012; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10013; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10014; GFX10-CU-NEXT:    s_endpgm
10015;
10016; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
10017; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10018; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10019; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10020; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10021; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10022; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10023; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10024; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10025; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10026; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10027; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10028; SKIP-CACHE-INV-NEXT:    s_endpgm
10029;
10030; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
10031; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10032; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10033; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10034; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10035; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10036; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10037; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10038; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10039;
10040; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
10041; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10042; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10043; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10044; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10045; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10046; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10047; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10048; GFX90A-TGSPLIT-NEXT:    s_endpgm
10049;
10050; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
10051; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10052; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10053; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10054; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10055; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10056; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10057; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10058; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10059;
10060; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
10061; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10062; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10063; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10064; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10065; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10066; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10067; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10068; GFX940-TGSPLIT-NEXT:    s_endpgm
10069;
10070; GFX11-WGP-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
10071; GFX11-WGP:       ; %bb.0: ; %entry
10072; GFX11-WGP-NEXT:    s_clause 0x1
10073; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10074; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10075; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10076; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10077; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10078; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10079; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10080; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10081; GFX11-WGP-NEXT:    s_endpgm
10082;
10083; GFX11-CU-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
10084; GFX11-CU:       ; %bb.0: ; %entry
10085; GFX11-CU-NEXT:    s_clause 0x1
10086; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10087; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10088; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10089; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10090; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10091; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10092; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10093; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10094; GFX11-CU-NEXT:    s_endpgm
10095    i32* %out, i32 %in, i32 %old) {
10096entry:
10097  %gep = getelementptr i32, i32* %out, i32 4
10098  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release seq_cst
10099  ret void
10100}
10101
10102define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
10103; GFX7-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10104; GFX7:       ; %bb.0: ; %entry
10105; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10106; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10107; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10108; GFX7-NEXT:    s_add_u32 s0, s0, 16
10109; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10110; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10111; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10112; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10113; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10114; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10115; GFX7-NEXT:    s_endpgm
10116;
10117; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10118; GFX10-WGP:       ; %bb.0: ; %entry
10119; GFX10-WGP-NEXT:    s_clause 0x1
10120; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10121; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10122; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10123; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10124; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10125; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10126; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10127; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10128; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10129; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10130; GFX10-WGP-NEXT:    s_endpgm
10131;
10132; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10133; GFX10-CU:       ; %bb.0: ; %entry
10134; GFX10-CU-NEXT:    s_clause 0x1
10135; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10136; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10137; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10138; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10139; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10140; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10141; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10142; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10143; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10144; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10145; GFX10-CU-NEXT:    s_endpgm
10146;
10147; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10148; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10149; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10150; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10151; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10152; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10153; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10154; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10155; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10156; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10157; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10158; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10159; SKIP-CACHE-INV-NEXT:    s_endpgm
10160;
10161; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10162; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10163; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10164; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10165; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10166; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10167; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10168; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10169; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10170;
10171; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10172; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10173; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10174; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10175; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10176; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10177; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10178; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10179; GFX90A-TGSPLIT-NEXT:    s_endpgm
10180;
10181; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10182; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10183; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10184; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10185; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10186; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10187; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10188; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10189; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10190;
10191; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10192; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10193; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10194; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10195; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10196; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10197; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10198; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10199; GFX940-TGSPLIT-NEXT:    s_endpgm
10200;
10201; GFX11-WGP-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10202; GFX11-WGP:       ; %bb.0: ; %entry
10203; GFX11-WGP-NEXT:    s_clause 0x1
10204; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10205; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10206; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10207; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10208; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10209; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10210; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10211; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10212; GFX11-WGP-NEXT:    s_endpgm
10213;
10214; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10215; GFX11-CU:       ; %bb.0: ; %entry
10216; GFX11-CU-NEXT:    s_clause 0x1
10217; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10218; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10219; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10220; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10221; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10222; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10223; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10224; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10225; GFX11-CU-NEXT:    s_endpgm
10226    i32* %out, i32 %in, i32 %old) {
10227entry:
10228  %gep = getelementptr i32, i32* %out, i32 4
10229  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel seq_cst
10230  ret void
10231}
10232
10233define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
10234; GFX7-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10235; GFX7:       ; %bb.0: ; %entry
10236; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10237; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10238; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10239; GFX7-NEXT:    s_add_u32 s0, s0, 16
10240; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10241; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10242; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10243; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10244; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10245; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10246; GFX7-NEXT:    s_endpgm
10247;
10248; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10249; GFX10-WGP:       ; %bb.0: ; %entry
10250; GFX10-WGP-NEXT:    s_clause 0x1
10251; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10252; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10253; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10254; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10255; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10256; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10257; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10258; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10259; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10260; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10261; GFX10-WGP-NEXT:    s_endpgm
10262;
10263; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10264; GFX10-CU:       ; %bb.0: ; %entry
10265; GFX10-CU-NEXT:    s_clause 0x1
10266; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10267; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10268; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10269; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10270; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10271; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10272; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10273; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10274; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10275; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10276; GFX10-CU-NEXT:    s_endpgm
10277;
10278; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10279; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10280; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10281; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10282; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10283; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10284; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10285; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10286; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10287; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10288; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10289; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10290; SKIP-CACHE-INV-NEXT:    s_endpgm
10291;
10292; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10293; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10294; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10295; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10296; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10297; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10298; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10299; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10300; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10301;
10302; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10303; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10304; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10305; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10306; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10307; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10308; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10309; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10310; GFX90A-TGSPLIT-NEXT:    s_endpgm
10311;
10312; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10313; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10314; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10315; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10316; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10317; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10318; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10319; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10320; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10321;
10322; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10323; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10324; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10325; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10326; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10327; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10328; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10329; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10330; GFX940-TGSPLIT-NEXT:    s_endpgm
10331;
10332; GFX11-WGP-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10333; GFX11-WGP:       ; %bb.0: ; %entry
10334; GFX11-WGP-NEXT:    s_clause 0x1
10335; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10336; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10337; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10338; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10339; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10340; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10341; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10342; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10343; GFX11-WGP-NEXT:    s_endpgm
10344;
10345; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10346; GFX11-CU:       ; %bb.0: ; %entry
10347; GFX11-CU-NEXT:    s_clause 0x1
10348; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10349; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10350; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10351; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10352; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10353; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10354; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10355; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10356; GFX11-CU-NEXT:    s_endpgm
10357    i32* %out, i32 %in, i32 %old) {
10358entry:
10359  %gep = getelementptr i32, i32* %out, i32 4
10360  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
10361  ret void
10362}
10363
10364define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg(
10365; GFX7-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10366; GFX7:       ; %bb.0: ; %entry
10367; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10368; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10369; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10370; GFX7-NEXT:    s_add_u32 s4, s0, 16
10371; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10372; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10373; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10374; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10375; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10376; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10377; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10378; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10379; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10380; GFX7-NEXT:    flat_store_dword v[0:1], v2
10381; GFX7-NEXT:    s_endpgm
10382;
10383; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10384; GFX10-WGP:       ; %bb.0: ; %entry
10385; GFX10-WGP-NEXT:    s_clause 0x1
10386; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10387; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10388; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10389; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10390; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10391; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10392; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10393; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10394; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10395; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10396; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10397; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10398; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10399; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10400; GFX10-WGP-NEXT:    s_endpgm
10401;
10402; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10403; GFX10-CU:       ; %bb.0: ; %entry
10404; GFX10-CU-NEXT:    s_clause 0x1
10405; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10406; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10407; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10408; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10409; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10410; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10411; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10412; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10413; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10414; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10415; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10416; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10417; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10418; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10419; GFX10-CU-NEXT:    s_endpgm
10420;
10421; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10422; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10423; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10424; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10425; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10426; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10427; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10428; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10429; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10430; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10431; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10432; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10433; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10434; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10435; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10436; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10437; SKIP-CACHE-INV-NEXT:    s_endpgm
10438;
10439; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10440; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10441; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10442; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10443; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10444; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10445; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10446; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10447; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10448; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10449; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10450;
10451; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10452; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10453; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10454; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10455; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10456; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10457; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10458; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10459; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10460; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10461; GFX90A-TGSPLIT-NEXT:    s_endpgm
10462;
10463; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10464; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10465; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10466; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10467; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10468; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10469; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10470; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10471; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10472; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10473; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10474;
10475; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10476; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10477; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10478; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10479; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10480; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10481; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10482; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10483; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10484; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10485; GFX940-TGSPLIT-NEXT:    s_endpgm
10486;
10487; GFX11-WGP-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10488; GFX11-WGP:       ; %bb.0: ; %entry
10489; GFX11-WGP-NEXT:    s_clause 0x1
10490; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10491; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10492; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10493; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10494; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10495; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10496; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10497; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10498; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10499; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
10500; GFX11-WGP-NEXT:    s_endpgm
10501;
10502; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10503; GFX11-CU:       ; %bb.0: ; %entry
10504; GFX11-CU-NEXT:    s_clause 0x1
10505; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10506; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10507; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10508; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10509; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10510; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10511; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10512; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10513; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10514; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
10515; GFX11-CU-NEXT:    s_endpgm
10516    i32* %out, i32 %in, i32 %old) {
10517entry:
10518  %gep = getelementptr i32, i32* %out, i32 4
10519  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
10520  %val0 = extractvalue { i32, i1 } %val, 0
10521  store i32 %val0, i32* %out, align 4
10522  ret void
10523}
10524
10525define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg(
10526; GFX7-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10527; GFX7:       ; %bb.0: ; %entry
10528; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10529; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10530; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10531; GFX7-NEXT:    s_add_u32 s4, s0, 16
10532; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10533; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10534; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10535; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10536; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10537; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10538; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10539; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10540; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10541; GFX7-NEXT:    flat_store_dword v[0:1], v2
10542; GFX7-NEXT:    s_endpgm
10543;
10544; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10545; GFX10-WGP:       ; %bb.0: ; %entry
10546; GFX10-WGP-NEXT:    s_clause 0x1
10547; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10548; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10549; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10550; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10551; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10552; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10553; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10554; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10555; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10556; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10557; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10558; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10559; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10560; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10561; GFX10-WGP-NEXT:    s_endpgm
10562;
10563; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10564; GFX10-CU:       ; %bb.0: ; %entry
10565; GFX10-CU-NEXT:    s_clause 0x1
10566; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10567; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10568; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10569; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10570; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10571; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10572; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10573; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10574; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10575; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10576; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10577; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10578; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10579; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10580; GFX10-CU-NEXT:    s_endpgm
10581;
10582; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10583; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10584; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10585; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10586; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10587; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10588; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10589; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10590; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10591; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10592; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10593; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10594; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10595; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10596; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10597; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10598; SKIP-CACHE-INV-NEXT:    s_endpgm
10599;
10600; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10601; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10602; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10603; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10604; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10605; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10606; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10607; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10608; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10609; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10610; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10611;
10612; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10613; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10614; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10615; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10616; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10617; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10618; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10619; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10620; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10621; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10622; GFX90A-TGSPLIT-NEXT:    s_endpgm
10623;
10624; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10625; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10626; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10627; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10628; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10629; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10630; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10631; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10632; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10633; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10634; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10635;
10636; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10637; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10638; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10639; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10640; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10641; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10642; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10643; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10644; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10645; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10646; GFX940-TGSPLIT-NEXT:    s_endpgm
10647;
10648; GFX11-WGP-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10649; GFX11-WGP:       ; %bb.0: ; %entry
10650; GFX11-WGP-NEXT:    s_clause 0x1
10651; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10652; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10653; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10654; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10655; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10656; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10657; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10658; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10659; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10660; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
10661; GFX11-WGP-NEXT:    s_endpgm
10662;
10663; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10664; GFX11-CU:       ; %bb.0: ; %entry
10665; GFX11-CU-NEXT:    s_clause 0x1
10666; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10667; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10668; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10669; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10670; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10671; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10672; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10673; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10674; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10675; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
10676; GFX11-CU-NEXT:    s_endpgm
10677    i32* %out, i32 %in, i32 %old) {
10678entry:
10679  %gep = getelementptr i32, i32* %out, i32 4
10680  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
10681  %val0 = extractvalue { i32, i1 } %val, 0
10682  store i32 %val0, i32* %out, align 4
10683  ret void
10684}
10685
10686define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_ret_cmpxchg(
10687; GFX7-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
10688; GFX7:       ; %bb.0: ; %entry
10689; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10690; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10691; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10692; GFX7-NEXT:    s_add_u32 s4, s0, 16
10693; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10694; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10695; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10696; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10697; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10698; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10699; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10700; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10701; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10702; GFX7-NEXT:    flat_store_dword v[0:1], v2
10703; GFX7-NEXT:    s_endpgm
10704;
10705; GFX10-WGP-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
10706; GFX10-WGP:       ; %bb.0: ; %entry
10707; GFX10-WGP-NEXT:    s_clause 0x1
10708; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10709; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10710; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10711; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10712; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10713; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10714; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10715; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10716; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10717; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10718; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10719; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10720; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10721; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10722; GFX10-WGP-NEXT:    s_endpgm
10723;
10724; GFX10-CU-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
10725; GFX10-CU:       ; %bb.0: ; %entry
10726; GFX10-CU-NEXT:    s_clause 0x1
10727; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10728; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10729; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10730; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10731; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10732; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10733; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10734; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10735; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10736; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10737; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10738; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10739; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10740; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10741; GFX10-CU-NEXT:    s_endpgm
10742;
10743; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
10744; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10745; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10746; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10747; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10748; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10749; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10750; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10751; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10752; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10753; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10754; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10755; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10756; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10757; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10758; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10759; SKIP-CACHE-INV-NEXT:    s_endpgm
10760;
10761; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
10762; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10763; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10764; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10765; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10766; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10767; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10768; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10769; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10770; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10771; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10772;
10773; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
10774; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10775; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10776; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10777; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10778; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10779; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10780; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10781; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10782; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10783; GFX90A-TGSPLIT-NEXT:    s_endpgm
10784;
10785; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
10786; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10787; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10788; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10789; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10790; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10791; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10792; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10793; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10794; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10795; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10796;
10797; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
10798; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10799; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10800; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10801; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10802; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10803; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10804; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10805; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10806; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10807; GFX940-TGSPLIT-NEXT:    s_endpgm
10808;
10809; GFX11-WGP-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
10810; GFX11-WGP:       ; %bb.0: ; %entry
10811; GFX11-WGP-NEXT:    s_clause 0x1
10812; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10813; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10814; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10815; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10816; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10817; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10818; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10819; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10820; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10821; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
10822; GFX11-WGP-NEXT:    s_endpgm
10823;
10824; GFX11-CU-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
10825; GFX11-CU:       ; %bb.0: ; %entry
10826; GFX11-CU-NEXT:    s_clause 0x1
10827; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10828; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10829; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10830; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10831; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10832; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10833; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10834; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10835; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10836; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
10837; GFX11-CU-NEXT:    s_endpgm
10838    i32* %out, i32 %in, i32 %old) {
10839entry:
10840  %gep = getelementptr i32, i32* %out, i32 4
10841  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
10842  %val0 = extractvalue { i32, i1 } %val, 0
10843  store i32 %val0, i32* %out, align 4
10844  ret void
10845}
10846
10847define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg(
10848; GFX7-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10849; GFX7:       ; %bb.0: ; %entry
10850; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10851; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10852; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10853; GFX7-NEXT:    s_add_u32 s4, s0, 16
10854; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10855; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10856; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10857; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10858; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10859; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10860; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10861; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10862; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10863; GFX7-NEXT:    flat_store_dword v[0:1], v2
10864; GFX7-NEXT:    s_endpgm
10865;
10866; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10867; GFX10-WGP:       ; %bb.0: ; %entry
10868; GFX10-WGP-NEXT:    s_clause 0x1
10869; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10870; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10871; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10872; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10873; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10874; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10875; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10876; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10877; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10878; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10879; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10880; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10881; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10882; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10883; GFX10-WGP-NEXT:    s_endpgm
10884;
10885; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10886; GFX10-CU:       ; %bb.0: ; %entry
10887; GFX10-CU-NEXT:    s_clause 0x1
10888; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10889; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10890; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10891; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10892; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10893; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10894; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10895; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10896; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10897; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10898; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10899; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10900; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10901; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10902; GFX10-CU-NEXT:    s_endpgm
10903;
10904; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10905; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10906; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10907; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10908; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10909; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10910; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10911; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10912; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10913; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10914; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10915; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10916; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10917; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10918; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10919; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10920; SKIP-CACHE-INV-NEXT:    s_endpgm
10921;
10922; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10923; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10924; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10925; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10926; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10927; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10928; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10929; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10930; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10931; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10932; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10933;
10934; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10935; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10936; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10937; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10938; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10939; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10940; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10941; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10942; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10943; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10944; GFX90A-TGSPLIT-NEXT:    s_endpgm
10945;
10946; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10947; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10948; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10949; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10950; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10951; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10952; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10953; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10954; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10955; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10956; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10957;
10958; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10959; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10960; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10961; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10962; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10963; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10964; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10965; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10966; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10967; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10968; GFX940-TGSPLIT-NEXT:    s_endpgm
10969;
10970; GFX11-WGP-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10971; GFX11-WGP:       ; %bb.0: ; %entry
10972; GFX11-WGP-NEXT:    s_clause 0x1
10973; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10974; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10975; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10976; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10977; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10978; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10979; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10980; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10981; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10982; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
10983; GFX11-WGP-NEXT:    s_endpgm
10984;
10985; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10986; GFX11-CU:       ; %bb.0: ; %entry
10987; GFX11-CU-NEXT:    s_clause 0x1
10988; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10989; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10990; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10991; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10992; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10993; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10994; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10995; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10996; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10997; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
10998; GFX11-CU-NEXT:    s_endpgm
10999    i32* %out, i32 %in, i32 %old) {
11000entry:
11001  %gep = getelementptr i32, i32* %out, i32 4
11002  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
11003  %val0 = extractvalue { i32, i1 } %val, 0
11004  store i32 %val0, i32* %out, align 4
11005  ret void
11006}
11007
11008define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg(
11009; GFX7-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11010; GFX7:       ; %bb.0: ; %entry
11011; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11012; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11013; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11014; GFX7-NEXT:    s_add_u32 s4, s0, 16
11015; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11016; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11017; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11018; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11019; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11020; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11021; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11022; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11023; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11024; GFX7-NEXT:    flat_store_dword v[0:1], v2
11025; GFX7-NEXT:    s_endpgm
11026;
11027; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11028; GFX10-WGP:       ; %bb.0: ; %entry
11029; GFX10-WGP-NEXT:    s_clause 0x1
11030; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11031; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11032; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11033; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11034; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11035; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11036; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11037; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11038; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11039; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11040; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11041; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11042; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11043; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11044; GFX10-WGP-NEXT:    s_endpgm
11045;
11046; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11047; GFX10-CU:       ; %bb.0: ; %entry
11048; GFX10-CU-NEXT:    s_clause 0x1
11049; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11050; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11051; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11052; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11053; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11054; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11055; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11056; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11057; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11058; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11059; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11060; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11061; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11062; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11063; GFX10-CU-NEXT:    s_endpgm
11064;
11065; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11066; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11067; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11068; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11069; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11070; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11071; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11072; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11073; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11074; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11075; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11076; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11077; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11078; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11079; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11080; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11081; SKIP-CACHE-INV-NEXT:    s_endpgm
11082;
11083; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11084; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11085; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11086; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11087; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11088; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11089; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11090; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11091; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11092; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11093; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11094;
11095; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11096; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11097; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11098; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11099; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11100; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11101; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11102; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11103; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11104; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11105; GFX90A-TGSPLIT-NEXT:    s_endpgm
11106;
11107; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11108; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11109; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11110; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11111; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11112; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11113; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11114; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11115; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11116; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11117; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11118;
11119; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11120; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11121; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11122; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11123; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11124; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11125; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11126; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11127; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11128; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11129; GFX940-TGSPLIT-NEXT:    s_endpgm
11130;
11131; GFX11-WGP-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11132; GFX11-WGP:       ; %bb.0: ; %entry
11133; GFX11-WGP-NEXT:    s_clause 0x1
11134; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11135; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11136; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11137; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11138; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11139; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11140; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11141; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11142; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11143; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11144; GFX11-WGP-NEXT:    s_endpgm
11145;
11146; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11147; GFX11-CU:       ; %bb.0: ; %entry
11148; GFX11-CU-NEXT:    s_clause 0x1
11149; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11150; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11151; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11152; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11153; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11154; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11155; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11156; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11157; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11158; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11159; GFX11-CU-NEXT:    s_endpgm
11160    i32* %out, i32 %in, i32 %old) {
11161entry:
11162  %gep = getelementptr i32, i32* %out, i32 4
11163  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
11164  %val0 = extractvalue { i32, i1 } %val, 0
11165  store i32 %val0, i32* %out, align 4
11166  ret void
11167}
11168
11169define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg(
11170; GFX7-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11171; GFX7:       ; %bb.0: ; %entry
11172; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11173; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11174; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11175; GFX7-NEXT:    s_add_u32 s4, s0, 16
11176; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11177; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11178; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11179; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11180; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11181; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11182; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11183; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11184; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11185; GFX7-NEXT:    flat_store_dword v[0:1], v2
11186; GFX7-NEXT:    s_endpgm
11187;
11188; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11189; GFX10-WGP:       ; %bb.0: ; %entry
11190; GFX10-WGP-NEXT:    s_clause 0x1
11191; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11192; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11193; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11194; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11195; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11196; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11197; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11198; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11199; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11200; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11201; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11202; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11203; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11204; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11205; GFX10-WGP-NEXT:    s_endpgm
11206;
11207; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11208; GFX10-CU:       ; %bb.0: ; %entry
11209; GFX10-CU-NEXT:    s_clause 0x1
11210; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11211; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11212; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11213; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11214; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11215; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11216; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11217; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11218; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11219; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11220; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11221; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11222; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11223; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11224; GFX10-CU-NEXT:    s_endpgm
11225;
11226; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11227; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11228; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11229; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11230; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11231; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11232; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11233; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11234; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11235; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11236; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11237; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11238; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11239; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11240; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11241; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11242; SKIP-CACHE-INV-NEXT:    s_endpgm
11243;
11244; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11245; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11246; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11247; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11248; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11249; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11250; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11251; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11252; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11253; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11254; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11255;
11256; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11257; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11258; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11259; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11260; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11261; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11262; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11263; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11264; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11265; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11266; GFX90A-TGSPLIT-NEXT:    s_endpgm
11267;
11268; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11269; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11270; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11271; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11272; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11273; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11274; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11275; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11276; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11277; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11278; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11279;
11280; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11281; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11282; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11283; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11284; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11285; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11286; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11287; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11288; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11289; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11290; GFX940-TGSPLIT-NEXT:    s_endpgm
11291;
11292; GFX11-WGP-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11293; GFX11-WGP:       ; %bb.0: ; %entry
11294; GFX11-WGP-NEXT:    s_clause 0x1
11295; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11296; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11297; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11298; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11299; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11300; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11301; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11302; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11303; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11304; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11305; GFX11-WGP-NEXT:    s_endpgm
11306;
11307; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11308; GFX11-CU:       ; %bb.0: ; %entry
11309; GFX11-CU-NEXT:    s_clause 0x1
11310; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11311; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11312; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11313; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11314; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11315; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11316; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11317; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11318; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11319; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11320; GFX11-CU-NEXT:    s_endpgm
11321    i32* %out, i32 %in, i32 %old) {
11322entry:
11323  %gep = getelementptr i32, i32* %out, i32 4
11324  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic acquire
11325  %val0 = extractvalue { i32, i1 } %val, 0
11326  store i32 %val0, i32* %out, align 4
11327  ret void
11328}
11329
11330define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_ret_cmpxchg(
11331; GFX7-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11332; GFX7:       ; %bb.0: ; %entry
11333; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11334; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11335; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11336; GFX7-NEXT:    s_add_u32 s4, s0, 16
11337; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11338; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11339; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11340; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11341; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11342; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11343; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11344; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11345; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11346; GFX7-NEXT:    flat_store_dword v[0:1], v2
11347; GFX7-NEXT:    s_endpgm
11348;
11349; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11350; GFX10-WGP:       ; %bb.0: ; %entry
11351; GFX10-WGP-NEXT:    s_clause 0x1
11352; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11353; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11354; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11355; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11356; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11357; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11358; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11359; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11360; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11361; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11362; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11363; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11364; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11365; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11366; GFX10-WGP-NEXT:    s_endpgm
11367;
11368; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11369; GFX10-CU:       ; %bb.0: ; %entry
11370; GFX10-CU-NEXT:    s_clause 0x1
11371; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11372; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11373; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11374; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11375; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11376; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11377; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11378; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11379; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11380; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11381; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11382; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11383; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11384; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11385; GFX10-CU-NEXT:    s_endpgm
11386;
11387; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11388; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11389; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11390; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11391; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11392; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11393; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11394; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11395; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11396; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11397; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11398; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11399; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11400; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11401; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11402; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11403; SKIP-CACHE-INV-NEXT:    s_endpgm
11404;
11405; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11406; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11407; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11408; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11409; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11410; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11411; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11412; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11413; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11414; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11415; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11416;
11417; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11418; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11419; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11420; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11421; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11422; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11423; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11424; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11425; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11426; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11427; GFX90A-TGSPLIT-NEXT:    s_endpgm
11428;
11429; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11430; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11431; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11432; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11433; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11434; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11435; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11436; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11437; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11438; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11439; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11440;
11441; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11442; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11443; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11444; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11445; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11446; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11447; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11448; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11449; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11450; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11451; GFX940-TGSPLIT-NEXT:    s_endpgm
11452;
11453; GFX11-WGP-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11454; GFX11-WGP:       ; %bb.0: ; %entry
11455; GFX11-WGP-NEXT:    s_clause 0x1
11456; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11457; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11458; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11459; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11460; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11461; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11462; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11463; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11464; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11465; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11466; GFX11-WGP-NEXT:    s_endpgm
11467;
11468; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11469; GFX11-CU:       ; %bb.0: ; %entry
11470; GFX11-CU-NEXT:    s_clause 0x1
11471; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11472; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11473; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11474; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11475; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11476; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11477; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11478; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11479; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11480; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11481; GFX11-CU-NEXT:    s_endpgm
11482    i32* %out, i32 %in, i32 %old) {
11483entry:
11484  %gep = getelementptr i32, i32* %out, i32 4
11485  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
11486  %val0 = extractvalue { i32, i1 } %val, 0
11487  store i32 %val0, i32* %out, align 4
11488  ret void
11489}
11490
11491define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_ret_cmpxchg(
11492; GFX7-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
11493; GFX7:       ; %bb.0: ; %entry
11494; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11495; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11496; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11497; GFX7-NEXT:    s_add_u32 s4, s0, 16
11498; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11499; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11500; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11501; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11502; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11503; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11504; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11505; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11506; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11507; GFX7-NEXT:    flat_store_dword v[0:1], v2
11508; GFX7-NEXT:    s_endpgm
11509;
11510; GFX10-WGP-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
11511; GFX10-WGP:       ; %bb.0: ; %entry
11512; GFX10-WGP-NEXT:    s_clause 0x1
11513; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11514; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11515; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11516; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11517; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11518; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11519; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11520; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11521; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11522; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11523; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11524; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11525; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11526; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11527; GFX10-WGP-NEXT:    s_endpgm
11528;
11529; GFX10-CU-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
11530; GFX10-CU:       ; %bb.0: ; %entry
11531; GFX10-CU-NEXT:    s_clause 0x1
11532; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11533; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11534; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11535; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11536; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11537; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11538; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11539; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11540; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11541; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11542; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11543; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11544; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11545; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11546; GFX10-CU-NEXT:    s_endpgm
11547;
11548; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
11549; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11550; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11551; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11552; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11553; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11554; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11555; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11556; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11557; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11558; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11559; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11560; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11561; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11562; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11563; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11564; SKIP-CACHE-INV-NEXT:    s_endpgm
11565;
11566; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
11567; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11568; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11569; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11570; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11571; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11572; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11573; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11574; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11575; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11576; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11577;
11578; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
11579; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11580; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11581; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11582; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11583; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11584; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11585; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11586; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11587; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11588; GFX90A-TGSPLIT-NEXT:    s_endpgm
11589;
11590; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
11591; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11592; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11593; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11594; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11595; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11596; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11597; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11598; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11599; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11600; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11601;
11602; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
11603; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11604; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11605; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11606; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11607; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11608; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11609; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11610; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11611; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11612; GFX940-TGSPLIT-NEXT:    s_endpgm
11613;
11614; GFX11-WGP-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
11615; GFX11-WGP:       ; %bb.0: ; %entry
11616; GFX11-WGP-NEXT:    s_clause 0x1
11617; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11618; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11619; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11620; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11621; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11622; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11623; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11624; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11625; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11626; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11627; GFX11-WGP-NEXT:    s_endpgm
11628;
11629; GFX11-CU-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
11630; GFX11-CU:       ; %bb.0: ; %entry
11631; GFX11-CU-NEXT:    s_clause 0x1
11632; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11633; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11634; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11635; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11636; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11637; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11638; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11639; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11640; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11641; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11642; GFX11-CU-NEXT:    s_endpgm
11643    i32* %out, i32 %in, i32 %old) {
11644entry:
11645  %gep = getelementptr i32, i32* %out, i32 4
11646  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
11647  %val0 = extractvalue { i32, i1 } %val, 0
11648  store i32 %val0, i32* %out, align 4
11649  ret void
11650}
11651
11652define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
11653; GFX7-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11654; GFX7:       ; %bb.0: ; %entry
11655; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11656; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11657; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11658; GFX7-NEXT:    s_add_u32 s4, s0, 16
11659; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11660; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11661; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11662; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11663; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11664; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11665; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11666; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11667; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11668; GFX7-NEXT:    flat_store_dword v[0:1], v2
11669; GFX7-NEXT:    s_endpgm
11670;
11671; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11672; GFX10-WGP:       ; %bb.0: ; %entry
11673; GFX10-WGP-NEXT:    s_clause 0x1
11674; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11675; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11676; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11677; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11678; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11679; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11680; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11681; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11682; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11683; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11684; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11685; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11686; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11687; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11688; GFX10-WGP-NEXT:    s_endpgm
11689;
11690; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11691; GFX10-CU:       ; %bb.0: ; %entry
11692; GFX10-CU-NEXT:    s_clause 0x1
11693; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11694; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11695; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11696; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11697; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11698; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11699; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11700; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11701; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11702; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11703; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11704; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11705; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11706; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11707; GFX10-CU-NEXT:    s_endpgm
11708;
11709; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11710; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11711; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11712; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11713; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11714; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11715; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11716; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11717; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11718; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11719; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11720; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11721; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11722; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11723; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11724; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11725; SKIP-CACHE-INV-NEXT:    s_endpgm
11726;
11727; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11728; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11729; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11730; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11731; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11732; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11733; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11734; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11735; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11736; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11737; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11738;
11739; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11740; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11741; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11742; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11743; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11744; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11745; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11746; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11747; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11748; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11749; GFX90A-TGSPLIT-NEXT:    s_endpgm
11750;
11751; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11752; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11753; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11754; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11755; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11756; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11757; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11758; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11759; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11760; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11761; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11762;
11763; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11764; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11765; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11766; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11767; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11768; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11769; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11770; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11771; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11772; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11773; GFX940-TGSPLIT-NEXT:    s_endpgm
11774;
11775; GFX11-WGP-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11776; GFX11-WGP:       ; %bb.0: ; %entry
11777; GFX11-WGP-NEXT:    s_clause 0x1
11778; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11779; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11780; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11781; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11782; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11783; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11784; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11785; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11786; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11787; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11788; GFX11-WGP-NEXT:    s_endpgm
11789;
11790; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11791; GFX11-CU:       ; %bb.0: ; %entry
11792; GFX11-CU-NEXT:    s_clause 0x1
11793; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11794; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11795; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11796; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11797; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11798; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11799; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11800; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11801; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11802; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11803; GFX11-CU-NEXT:    s_endpgm
11804    i32* %out, i32 %in, i32 %old) {
11805entry:
11806  %gep = getelementptr i32, i32* %out, i32 4
11807  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
11808  %val0 = extractvalue { i32, i1 } %val, 0
11809  store i32 %val0, i32* %out, align 4
11810  ret void
11811}
11812
11813define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
11814; GFX7-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11815; GFX7:       ; %bb.0: ; %entry
11816; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11817; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11818; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11819; GFX7-NEXT:    s_add_u32 s4, s0, 16
11820; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11821; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11822; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11823; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11824; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11825; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11826; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11827; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11828; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11829; GFX7-NEXT:    flat_store_dword v[0:1], v2
11830; GFX7-NEXT:    s_endpgm
11831;
11832; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11833; GFX10-WGP:       ; %bb.0: ; %entry
11834; GFX10-WGP-NEXT:    s_clause 0x1
11835; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11836; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11837; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11838; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11839; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11840; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11841; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11842; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11843; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11844; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11845; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11846; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11847; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11848; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11849; GFX10-WGP-NEXT:    s_endpgm
11850;
11851; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11852; GFX10-CU:       ; %bb.0: ; %entry
11853; GFX10-CU-NEXT:    s_clause 0x1
11854; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11855; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11856; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11857; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11858; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11859; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11860; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11861; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11862; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11863; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11864; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11865; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11866; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11867; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11868; GFX10-CU-NEXT:    s_endpgm
11869;
11870; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11871; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11872; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11873; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11874; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11875; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11876; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11877; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11878; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11879; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11880; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11881; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11882; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11883; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11884; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11885; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11886; SKIP-CACHE-INV-NEXT:    s_endpgm
11887;
11888; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11889; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11890; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11891; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11892; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11893; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11894; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11895; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11896; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11897; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11898; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11899;
11900; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11901; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11902; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11903; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11904; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11905; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11906; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11907; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11908; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11909; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11910; GFX90A-TGSPLIT-NEXT:    s_endpgm
11911;
11912; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11913; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11914; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11915; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11916; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11917; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11918; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11919; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11920; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11921; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11922; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11923;
11924; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11925; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11926; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11927; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11928; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11929; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11930; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11931; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11932; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11933; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11934; GFX940-TGSPLIT-NEXT:    s_endpgm
11935;
11936; GFX11-WGP-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11937; GFX11-WGP:       ; %bb.0: ; %entry
11938; GFX11-WGP-NEXT:    s_clause 0x1
11939; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11940; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11941; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11942; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11943; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11944; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11945; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11946; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11947; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11948; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
11949; GFX11-WGP-NEXT:    s_endpgm
11950;
11951; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11952; GFX11-CU:       ; %bb.0: ; %entry
11953; GFX11-CU-NEXT:    s_clause 0x1
11954; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11955; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11956; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11957; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11958; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11959; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11960; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11961; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11962; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11963; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
11964; GFX11-CU-NEXT:    s_endpgm
11965    i32* %out, i32 %in, i32 %old) {
11966entry:
11967  %gep = getelementptr i32, i32* %out, i32 4
11968  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
11969  %val0 = extractvalue { i32, i1 } %val, 0
11970  store i32 %val0, i32* %out, align 4
11971  ret void
11972}
11973
11974define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg(
11975; GFX7-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
11976; GFX7:       ; %bb.0: ; %entry
11977; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11978; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11979; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11980; GFX7-NEXT:    s_add_u32 s4, s0, 16
11981; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11982; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11983; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11984; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11985; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11986; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11987; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11988; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11989; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
11990; GFX7-NEXT:    flat_store_dword v[0:1], v2
11991; GFX7-NEXT:    s_endpgm
11992;
11993; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
11994; GFX10-WGP:       ; %bb.0: ; %entry
11995; GFX10-WGP-NEXT:    s_clause 0x1
11996; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11997; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11998; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11999; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12000; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12001; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12002; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12003; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12004; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12005; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12006; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12007; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12008; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12009; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12010; GFX10-WGP-NEXT:    s_endpgm
12011;
12012; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12013; GFX10-CU:       ; %bb.0: ; %entry
12014; GFX10-CU-NEXT:    s_clause 0x1
12015; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12016; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12017; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12018; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12019; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12020; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12021; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12022; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12023; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12024; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12025; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12026; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12027; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12028; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12029; GFX10-CU-NEXT:    s_endpgm
12030;
12031; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12032; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12033; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12034; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12035; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12036; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12037; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12038; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12039; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12040; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12041; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12042; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12043; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12044; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12045; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12046; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12047; SKIP-CACHE-INV-NEXT:    s_endpgm
12048;
12049; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12050; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12051; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12052; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12053; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12054; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12055; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12056; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12057; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12058; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12059; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12060;
12061; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12062; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12063; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12064; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12065; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12066; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12067; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12068; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12069; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12070; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12071; GFX90A-TGSPLIT-NEXT:    s_endpgm
12072;
12073; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12074; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12075; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12076; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12077; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12078; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12079; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12080; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12081; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12082; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12083; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12084;
12085; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12086; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12087; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12088; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12089; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12090; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12091; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12092; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12093; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12094; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12095; GFX940-TGSPLIT-NEXT:    s_endpgm
12096;
12097; GFX11-WGP-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12098; GFX11-WGP:       ; %bb.0: ; %entry
12099; GFX11-WGP-NEXT:    s_clause 0x1
12100; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12101; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12102; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12103; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12104; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12105; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12106; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12107; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12108; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12109; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12110; GFX11-WGP-NEXT:    s_endpgm
12111;
12112; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12113; GFX11-CU:       ; %bb.0: ; %entry
12114; GFX11-CU-NEXT:    s_clause 0x1
12115; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12116; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12117; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12118; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12119; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12120; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12121; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12122; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12123; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12124; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12125; GFX11-CU-NEXT:    s_endpgm
12126    i32* %out, i32 %in, i32 %old) {
12127entry:
12128  %gep = getelementptr i32, i32* %out, i32 4
12129  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic seq_cst
12130  %val0 = extractvalue { i32, i1 } %val, 0
12131  store i32 %val0, i32* %out, align 4
12132  ret void
12133}
12134
12135define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg(
12136; GFX7-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12137; GFX7:       ; %bb.0: ; %entry
12138; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12139; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12140; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12141; GFX7-NEXT:    s_add_u32 s4, s0, 16
12142; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12143; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12144; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12145; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12146; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12147; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12148; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12149; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12150; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12151; GFX7-NEXT:    flat_store_dword v[0:1], v2
12152; GFX7-NEXT:    s_endpgm
12153;
12154; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12155; GFX10-WGP:       ; %bb.0: ; %entry
12156; GFX10-WGP-NEXT:    s_clause 0x1
12157; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12158; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12159; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12160; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12161; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12162; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12163; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12164; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12165; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12166; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12167; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12168; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12169; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12170; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12171; GFX10-WGP-NEXT:    s_endpgm
12172;
12173; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12174; GFX10-CU:       ; %bb.0: ; %entry
12175; GFX10-CU-NEXT:    s_clause 0x1
12176; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12177; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12178; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12179; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12180; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12181; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12182; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12183; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12184; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12185; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12186; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12187; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12188; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12189; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12190; GFX10-CU-NEXT:    s_endpgm
12191;
12192; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12193; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12194; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12195; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12196; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12197; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12198; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12199; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12200; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12201; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12202; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12203; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12204; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12205; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12206; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12207; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12208; SKIP-CACHE-INV-NEXT:    s_endpgm
12209;
12210; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12211; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12212; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12213; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12214; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12215; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12216; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12217; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12218; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12219; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12220; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12221;
12222; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12223; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12224; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12225; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12226; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12227; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12228; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12229; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12230; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12231; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12232; GFX90A-TGSPLIT-NEXT:    s_endpgm
12233;
12234; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12235; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12236; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12237; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12238; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12239; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12240; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12241; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12242; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12243; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12244; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12245;
12246; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12247; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12248; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12249; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12250; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12251; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12252; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12253; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12254; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12255; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12256; GFX940-TGSPLIT-NEXT:    s_endpgm
12257;
12258; GFX11-WGP-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12259; GFX11-WGP:       ; %bb.0: ; %entry
12260; GFX11-WGP-NEXT:    s_clause 0x1
12261; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12262; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12263; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12264; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12265; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12266; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12267; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12268; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12269; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12270; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12271; GFX11-WGP-NEXT:    s_endpgm
12272;
12273; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12274; GFX11-CU:       ; %bb.0: ; %entry
12275; GFX11-CU-NEXT:    s_clause 0x1
12276; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12277; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12278; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12279; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12280; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12281; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12282; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12283; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12284; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12285; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12286; GFX11-CU-NEXT:    s_endpgm
12287    i32* %out, i32 %in, i32 %old) {
12288entry:
12289  %gep = getelementptr i32, i32* %out, i32 4
12290  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire seq_cst
12291  %val0 = extractvalue { i32, i1 } %val, 0
12292  store i32 %val0, i32* %out, align 4
12293  ret void
12294}
12295
12296define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_ret_cmpxchg(
12297; GFX7-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12298; GFX7:       ; %bb.0: ; %entry
12299; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12300; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12301; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12302; GFX7-NEXT:    s_add_u32 s4, s0, 16
12303; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12304; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12305; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12306; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12307; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12308; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12309; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12310; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12311; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12312; GFX7-NEXT:    flat_store_dword v[0:1], v2
12313; GFX7-NEXT:    s_endpgm
12314;
12315; GFX10-WGP-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12316; GFX10-WGP:       ; %bb.0: ; %entry
12317; GFX10-WGP-NEXT:    s_clause 0x1
12318; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12319; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12320; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12321; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12322; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12323; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12324; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12325; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12326; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12327; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12328; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12329; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12330; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12331; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12332; GFX10-WGP-NEXT:    s_endpgm
12333;
12334; GFX10-CU-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12335; GFX10-CU:       ; %bb.0: ; %entry
12336; GFX10-CU-NEXT:    s_clause 0x1
12337; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12338; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12339; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12340; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12341; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12342; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12343; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12344; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12345; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12346; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12347; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12348; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12349; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12350; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12351; GFX10-CU-NEXT:    s_endpgm
12352;
12353; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12354; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12355; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12356; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12357; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12358; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12359; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12360; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12361; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12362; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12363; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12364; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12365; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12366; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12367; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12368; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12369; SKIP-CACHE-INV-NEXT:    s_endpgm
12370;
12371; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12372; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12373; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12374; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12375; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12376; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12377; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12378; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12379; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12380; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12381; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12382;
12383; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12384; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12385; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12386; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12387; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12388; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12389; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12390; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12391; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12392; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12393; GFX90A-TGSPLIT-NEXT:    s_endpgm
12394;
12395; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12396; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12397; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12398; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12399; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12400; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12401; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12402; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12403; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12404; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12405; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12406;
12407; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12408; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12409; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12410; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12411; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12412; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12413; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12414; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12415; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12416; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12417; GFX940-TGSPLIT-NEXT:    s_endpgm
12418;
12419; GFX11-WGP-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12420; GFX11-WGP:       ; %bb.0: ; %entry
12421; GFX11-WGP-NEXT:    s_clause 0x1
12422; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12423; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12424; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12425; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12426; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12427; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12428; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12429; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12430; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12431; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12432; GFX11-WGP-NEXT:    s_endpgm
12433;
12434; GFX11-CU-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12435; GFX11-CU:       ; %bb.0: ; %entry
12436; GFX11-CU-NEXT:    s_clause 0x1
12437; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12438; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12439; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12440; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12441; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12442; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12443; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12444; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12445; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12446; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12447; GFX11-CU-NEXT:    s_endpgm
12448    i32* %out, i32 %in, i32 %old) {
12449entry:
12450  %gep = getelementptr i32, i32* %out, i32 4
12451  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release seq_cst
12452  %val0 = extractvalue { i32, i1 } %val, 0
12453  store i32 %val0, i32* %out, align 4
12454  ret void
12455}
12456
12457define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg(
12458; GFX7-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12459; GFX7:       ; %bb.0: ; %entry
12460; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12461; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12462; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12463; GFX7-NEXT:    s_add_u32 s4, s0, 16
12464; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12465; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12466; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12467; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12468; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12469; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12470; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12471; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12472; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12473; GFX7-NEXT:    flat_store_dword v[0:1], v2
12474; GFX7-NEXT:    s_endpgm
12475;
12476; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12477; GFX10-WGP:       ; %bb.0: ; %entry
12478; GFX10-WGP-NEXT:    s_clause 0x1
12479; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12480; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12481; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12482; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12483; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12484; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12485; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12486; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12487; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12488; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12489; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12490; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12491; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12492; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12493; GFX10-WGP-NEXT:    s_endpgm
12494;
12495; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12496; GFX10-CU:       ; %bb.0: ; %entry
12497; GFX10-CU-NEXT:    s_clause 0x1
12498; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12499; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12500; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12501; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12502; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12503; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12504; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12505; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12506; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12507; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12508; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12509; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12510; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12511; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12512; GFX10-CU-NEXT:    s_endpgm
12513;
12514; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12515; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12516; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12517; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12518; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12519; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12520; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12521; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12522; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12523; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12524; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12525; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12526; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12527; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12528; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12529; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12530; SKIP-CACHE-INV-NEXT:    s_endpgm
12531;
12532; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12533; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12534; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12535; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12536; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12537; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12538; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12539; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12540; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12541; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12542; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12543;
12544; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12545; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12546; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12547; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12548; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12549; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12550; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12551; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12552; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12553; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12554; GFX90A-TGSPLIT-NEXT:    s_endpgm
12555;
12556; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12557; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12558; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12559; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12560; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12561; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12562; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12563; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12564; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12565; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12566; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12567;
12568; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12569; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12570; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12571; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12572; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12573; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12574; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12575; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12576; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12577; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12578; GFX940-TGSPLIT-NEXT:    s_endpgm
12579;
12580; GFX11-WGP-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12581; GFX11-WGP:       ; %bb.0: ; %entry
12582; GFX11-WGP-NEXT:    s_clause 0x1
12583; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12584; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12585; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12586; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12587; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12588; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12589; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12590; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12591; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12592; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12593; GFX11-WGP-NEXT:    s_endpgm
12594;
12595; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12596; GFX11-CU:       ; %bb.0: ; %entry
12597; GFX11-CU-NEXT:    s_clause 0x1
12598; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12599; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12600; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12601; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12602; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12603; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12604; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12605; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12606; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12607; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12608; GFX11-CU-NEXT:    s_endpgm
12609    i32* %out, i32 %in, i32 %old) {
12610entry:
12611  %gep = getelementptr i32, i32* %out, i32 4
12612  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel seq_cst
12613  %val0 = extractvalue { i32, i1 } %val, 0
12614  store i32 %val0, i32* %out, align 4
12615  ret void
12616}
12617
12618define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
12619; GFX7-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12620; GFX7:       ; %bb.0: ; %entry
12621; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12622; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12623; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12624; GFX7-NEXT:    s_add_u32 s4, s0, 16
12625; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12626; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12627; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12628; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12629; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12630; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12631; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12632; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12633; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12634; GFX7-NEXT:    flat_store_dword v[0:1], v2
12635; GFX7-NEXT:    s_endpgm
12636;
12637; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12638; GFX10-WGP:       ; %bb.0: ; %entry
12639; GFX10-WGP-NEXT:    s_clause 0x1
12640; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12641; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12642; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12643; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12644; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12645; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12646; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12647; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12648; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12649; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12650; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12651; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12652; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12653; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12654; GFX10-WGP-NEXT:    s_endpgm
12655;
12656; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12657; GFX10-CU:       ; %bb.0: ; %entry
12658; GFX10-CU-NEXT:    s_clause 0x1
12659; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12660; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12661; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12662; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12663; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12664; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12665; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12666; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12667; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12668; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12669; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12670; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12671; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12672; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12673; GFX10-CU-NEXT:    s_endpgm
12674;
12675; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12676; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12677; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12678; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12679; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12680; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12681; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12682; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12683; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12684; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12685; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12686; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12687; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12688; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12689; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12690; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12691; SKIP-CACHE-INV-NEXT:    s_endpgm
12692;
12693; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12694; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12695; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12696; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12697; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12698; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12699; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12700; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12701; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12702; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12703; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12704;
12705; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12706; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12707; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12708; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12709; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12710; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12711; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12712; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12713; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12714; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12715; GFX90A-TGSPLIT-NEXT:    s_endpgm
12716;
12717; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12718; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12719; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12720; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12721; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12722; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12723; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12724; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12725; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12726; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12727; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12728;
12729; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12730; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12731; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12732; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12733; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12734; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12735; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12736; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12737; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12738; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12739; GFX940-TGSPLIT-NEXT:    s_endpgm
12740;
12741; GFX11-WGP-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12742; GFX11-WGP:       ; %bb.0: ; %entry
12743; GFX11-WGP-NEXT:    s_clause 0x1
12744; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12745; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12746; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12747; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12748; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12749; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12750; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12751; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12752; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12753; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12754; GFX11-WGP-NEXT:    s_endpgm
12755;
12756; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12757; GFX11-CU:       ; %bb.0: ; %entry
12758; GFX11-CU-NEXT:    s_clause 0x1
12759; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12760; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12761; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12762; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12763; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12764; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12765; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12766; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12767; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12768; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12769; GFX11-CU-NEXT:    s_endpgm
12770    i32* %out, i32 %in, i32 %old) {
12771entry:
12772  %gep = getelementptr i32, i32* %out, i32 4
12773  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
12774  %val0 = extractvalue { i32, i1 } %val, 0
12775  store i32 %val0, i32* %out, align 4
12776  ret void
12777}
12778
12779