1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
5; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
6; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
8; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
9; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
10; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
11; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-WGP %s
12; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-CU %s
13
14define amdgpu_kernel void @local_singlethread_unordered_load(
15; GFX6-LABEL: local_singlethread_unordered_load:
16; GFX6:       ; %bb.0: ; %entry
17; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
18; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
19; GFX6-NEXT:    s_mov_b32 m0, -1
20; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
21; GFX6-NEXT:    v_mov_b32_e32 v0, s0
22; GFX6-NEXT:    ds_read_b32 v0, v0
23; GFX6-NEXT:    v_mov_b32_e32 v1, s1
24; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
25; GFX6-NEXT:    ds_write_b32 v1, v0
26; GFX6-NEXT:    s_endpgm
27;
28; GFX7-LABEL: local_singlethread_unordered_load:
29; GFX7:       ; %bb.0: ; %entry
30; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
31; GFX7-NEXT:    s_mov_b32 m0, -1
32; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
33; GFX7-NEXT:    v_mov_b32_e32 v0, s0
34; GFX7-NEXT:    ds_read_b32 v0, v0
35; GFX7-NEXT:    v_mov_b32_e32 v1, s1
36; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
37; GFX7-NEXT:    ds_write_b32 v1, v0
38; GFX7-NEXT:    s_endpgm
39;
40; GFX10-WGP-LABEL: local_singlethread_unordered_load:
41; GFX10-WGP:       ; %bb.0: ; %entry
42; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
43; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
44; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
45; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
46; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
47; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
48; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
49; GFX10-WGP-NEXT:    s_endpgm
50;
51; GFX10-CU-LABEL: local_singlethread_unordered_load:
52; GFX10-CU:       ; %bb.0: ; %entry
53; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
54; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
55; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
56; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
57; GFX10-CU-NEXT:    ds_read_b32 v0, v0
58; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
59; GFX10-CU-NEXT:    ds_write_b32 v1, v0
60; GFX10-CU-NEXT:    s_endpgm
61;
62; SKIP-CACHE-INV-LABEL: local_singlethread_unordered_load:
63; SKIP-CACHE-INV:       ; %bb.0: ; %entry
64; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
65; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
66; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
67; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
68; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
69; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
70; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
71; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
72; SKIP-CACHE-INV-NEXT:    s_endpgm
73;
74; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_unordered_load:
75; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
76; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
77; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
78; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
79; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
80; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
81; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
82; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
83; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
84;
85; GFX90A-TGSPLIT-LABEL: local_singlethread_unordered_load:
86; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
87; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
88; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
89; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
90; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
91; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
92; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
93; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
94; GFX90A-TGSPLIT-NEXT:    s_endpgm
95;
96; GFX940-NOTTGSPLIT-LABEL: local_singlethread_unordered_load:
97; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
98; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
99; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
100; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
101; GFX940-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
102; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
103; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
104; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
105; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
106;
107; GFX940-TGSPLIT-LABEL: local_singlethread_unordered_load:
108; GFX940-TGSPLIT:       ; %bb.0: ; %entry
109; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
110; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
111; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
112; GFX940-TGSPLIT-NEXT:    ds_read_b32 v0, v0
113; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
114; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
115; GFX940-TGSPLIT-NEXT:    ds_write_b32 v1, v0
116; GFX940-TGSPLIT-NEXT:    s_endpgm
117;
118; GFX11-WGP-LABEL: local_singlethread_unordered_load:
119; GFX11-WGP:       ; %bb.0: ; %entry
120; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
121; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
122; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
123; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
124; GFX11-WGP-NEXT:    ds_load_b32 v0, v0
125; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
126; GFX11-WGP-NEXT:    ds_store_b32 v1, v0
127; GFX11-WGP-NEXT:    s_endpgm
128;
129; GFX11-CU-LABEL: local_singlethread_unordered_load:
130; GFX11-CU:       ; %bb.0: ; %entry
131; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
132; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
133; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
134; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
135; GFX11-CU-NEXT:    ds_load_b32 v0, v0
136; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
137; GFX11-CU-NEXT:    ds_store_b32 v1, v0
138; GFX11-CU-NEXT:    s_endpgm
139    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
140entry:
141  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread") unordered, align 4
142  store i32 %val, i32 addrspace(3)* %out
143  ret void
144}
145
146define amdgpu_kernel void @local_singlethread_monotonic_load(
147; GFX6-LABEL: local_singlethread_monotonic_load:
148; GFX6:       ; %bb.0: ; %entry
149; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
150; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
151; GFX6-NEXT:    s_mov_b32 m0, -1
152; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
153; GFX6-NEXT:    v_mov_b32_e32 v0, s0
154; GFX6-NEXT:    ds_read_b32 v0, v0
155; GFX6-NEXT:    v_mov_b32_e32 v1, s1
156; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
157; GFX6-NEXT:    ds_write_b32 v1, v0
158; GFX6-NEXT:    s_endpgm
159;
160; GFX7-LABEL: local_singlethread_monotonic_load:
161; GFX7:       ; %bb.0: ; %entry
162; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
163; GFX7-NEXT:    s_mov_b32 m0, -1
164; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
165; GFX7-NEXT:    v_mov_b32_e32 v0, s0
166; GFX7-NEXT:    ds_read_b32 v0, v0
167; GFX7-NEXT:    v_mov_b32_e32 v1, s1
168; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
169; GFX7-NEXT:    ds_write_b32 v1, v0
170; GFX7-NEXT:    s_endpgm
171;
172; GFX10-WGP-LABEL: local_singlethread_monotonic_load:
173; GFX10-WGP:       ; %bb.0: ; %entry
174; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
175; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
176; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
177; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
178; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
179; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
180; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
181; GFX10-WGP-NEXT:    s_endpgm
182;
183; GFX10-CU-LABEL: local_singlethread_monotonic_load:
184; GFX10-CU:       ; %bb.0: ; %entry
185; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
186; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
187; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
188; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
189; GFX10-CU-NEXT:    ds_read_b32 v0, v0
190; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
191; GFX10-CU-NEXT:    ds_write_b32 v1, v0
192; GFX10-CU-NEXT:    s_endpgm
193;
194; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_load:
195; SKIP-CACHE-INV:       ; %bb.0: ; %entry
196; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
197; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
198; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
199; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
200; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
201; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
202; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
203; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
204; SKIP-CACHE-INV-NEXT:    s_endpgm
205;
206; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_load:
207; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
208; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
209; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
210; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
211; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
212; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
213; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
214; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
215; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
216;
217; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_load:
218; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
219; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
220; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
221; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
222; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
223; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
224; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
225; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
226; GFX90A-TGSPLIT-NEXT:    s_endpgm
227;
228; GFX940-NOTTGSPLIT-LABEL: local_singlethread_monotonic_load:
229; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
230; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
231; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
232; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
233; GFX940-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
234; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
235; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
236; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
237; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
238;
239; GFX940-TGSPLIT-LABEL: local_singlethread_monotonic_load:
240; GFX940-TGSPLIT:       ; %bb.0: ; %entry
241; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
242; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
243; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
244; GFX940-TGSPLIT-NEXT:    ds_read_b32 v0, v0
245; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
246; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
247; GFX940-TGSPLIT-NEXT:    ds_write_b32 v1, v0
248; GFX940-TGSPLIT-NEXT:    s_endpgm
249;
250; GFX11-WGP-LABEL: local_singlethread_monotonic_load:
251; GFX11-WGP:       ; %bb.0: ; %entry
252; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
253; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
254; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
255; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
256; GFX11-WGP-NEXT:    ds_load_b32 v0, v0
257; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
258; GFX11-WGP-NEXT:    ds_store_b32 v1, v0
259; GFX11-WGP-NEXT:    s_endpgm
260;
261; GFX11-CU-LABEL: local_singlethread_monotonic_load:
262; GFX11-CU:       ; %bb.0: ; %entry
263; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
264; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
265; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
266; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
267; GFX11-CU-NEXT:    ds_load_b32 v0, v0
268; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
269; GFX11-CU-NEXT:    ds_store_b32 v1, v0
270; GFX11-CU-NEXT:    s_endpgm
271    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
272entry:
273  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread") monotonic, align 4
274  store i32 %val, i32 addrspace(3)* %out
275  ret void
276}
277
278define amdgpu_kernel void @local_singlethread_acquire_load(
279; GFX6-LABEL: local_singlethread_acquire_load:
280; GFX6:       ; %bb.0: ; %entry
281; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
282; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
283; GFX6-NEXT:    s_mov_b32 m0, -1
284; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
285; GFX6-NEXT:    v_mov_b32_e32 v0, s0
286; GFX6-NEXT:    ds_read_b32 v0, v0
287; GFX6-NEXT:    v_mov_b32_e32 v1, s1
288; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
289; GFX6-NEXT:    ds_write_b32 v1, v0
290; GFX6-NEXT:    s_endpgm
291;
292; GFX7-LABEL: local_singlethread_acquire_load:
293; GFX7:       ; %bb.0: ; %entry
294; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
295; GFX7-NEXT:    s_mov_b32 m0, -1
296; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
297; GFX7-NEXT:    v_mov_b32_e32 v0, s0
298; GFX7-NEXT:    ds_read_b32 v0, v0
299; GFX7-NEXT:    v_mov_b32_e32 v1, s1
300; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
301; GFX7-NEXT:    ds_write_b32 v1, v0
302; GFX7-NEXT:    s_endpgm
303;
304; GFX10-WGP-LABEL: local_singlethread_acquire_load:
305; GFX10-WGP:       ; %bb.0: ; %entry
306; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
307; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
308; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
309; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
310; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
311; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
312; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
313; GFX10-WGP-NEXT:    s_endpgm
314;
315; GFX10-CU-LABEL: local_singlethread_acquire_load:
316; GFX10-CU:       ; %bb.0: ; %entry
317; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
318; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
319; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
320; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
321; GFX10-CU-NEXT:    ds_read_b32 v0, v0
322; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
323; GFX10-CU-NEXT:    ds_write_b32 v1, v0
324; GFX10-CU-NEXT:    s_endpgm
325;
326; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_load:
327; SKIP-CACHE-INV:       ; %bb.0: ; %entry
328; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
329; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
330; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
331; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
332; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
333; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
334; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
335; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
336; SKIP-CACHE-INV-NEXT:    s_endpgm
337;
338; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_load:
339; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
340; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
341; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
342; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
343; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
344; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
345; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
346; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
347; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
348;
349; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_load:
350; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
351; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
352; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
353; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
354; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
355; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
356; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
357; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
358; GFX90A-TGSPLIT-NEXT:    s_endpgm
359;
360; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acquire_load:
361; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
362; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
363; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
364; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
365; GFX940-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
366; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
367; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
368; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
369; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
370;
371; GFX940-TGSPLIT-LABEL: local_singlethread_acquire_load:
372; GFX940-TGSPLIT:       ; %bb.0: ; %entry
373; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
374; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
375; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
376; GFX940-TGSPLIT-NEXT:    ds_read_b32 v0, v0
377; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
378; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
379; GFX940-TGSPLIT-NEXT:    ds_write_b32 v1, v0
380; GFX940-TGSPLIT-NEXT:    s_endpgm
381;
382; GFX11-WGP-LABEL: local_singlethread_acquire_load:
383; GFX11-WGP:       ; %bb.0: ; %entry
384; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
385; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
386; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
387; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
388; GFX11-WGP-NEXT:    ds_load_b32 v0, v0
389; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
390; GFX11-WGP-NEXT:    ds_store_b32 v1, v0
391; GFX11-WGP-NEXT:    s_endpgm
392;
393; GFX11-CU-LABEL: local_singlethread_acquire_load:
394; GFX11-CU:       ; %bb.0: ; %entry
395; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
396; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
397; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
398; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
399; GFX11-CU-NEXT:    ds_load_b32 v0, v0
400; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
401; GFX11-CU-NEXT:    ds_store_b32 v1, v0
402; GFX11-CU-NEXT:    s_endpgm
403    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
404entry:
405  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread") acquire, align 4
406  store i32 %val, i32 addrspace(3)* %out
407  ret void
408}
409
410define amdgpu_kernel void @local_singlethread_seq_cst_load(
411; GFX6-LABEL: local_singlethread_seq_cst_load:
412; GFX6:       ; %bb.0: ; %entry
413; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
414; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
415; GFX6-NEXT:    s_mov_b32 m0, -1
416; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
417; GFX6-NEXT:    v_mov_b32_e32 v0, s0
418; GFX6-NEXT:    ds_read_b32 v0, v0
419; GFX6-NEXT:    v_mov_b32_e32 v1, s1
420; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
421; GFX6-NEXT:    ds_write_b32 v1, v0
422; GFX6-NEXT:    s_endpgm
423;
424; GFX7-LABEL: local_singlethread_seq_cst_load:
425; GFX7:       ; %bb.0: ; %entry
426; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
427; GFX7-NEXT:    s_mov_b32 m0, -1
428; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
429; GFX7-NEXT:    v_mov_b32_e32 v0, s0
430; GFX7-NEXT:    ds_read_b32 v0, v0
431; GFX7-NEXT:    v_mov_b32_e32 v1, s1
432; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
433; GFX7-NEXT:    ds_write_b32 v1, v0
434; GFX7-NEXT:    s_endpgm
435;
436; GFX10-WGP-LABEL: local_singlethread_seq_cst_load:
437; GFX10-WGP:       ; %bb.0: ; %entry
438; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
439; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
440; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
441; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
442; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
443; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
444; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
445; GFX10-WGP-NEXT:    s_endpgm
446;
447; GFX10-CU-LABEL: local_singlethread_seq_cst_load:
448; GFX10-CU:       ; %bb.0: ; %entry
449; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
450; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
451; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
452; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
453; GFX10-CU-NEXT:    ds_read_b32 v0, v0
454; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
455; GFX10-CU-NEXT:    ds_write_b32 v1, v0
456; GFX10-CU-NEXT:    s_endpgm
457;
458; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_load:
459; SKIP-CACHE-INV:       ; %bb.0: ; %entry
460; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
461; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
462; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
463; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
464; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
465; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
466; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
467; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
468; SKIP-CACHE-INV-NEXT:    s_endpgm
469;
470; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_load:
471; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
472; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
473; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
474; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
475; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
476; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
477; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
478; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
479; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
480;
481; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_load:
482; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
483; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
484; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
485; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
486; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
487; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
488; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
489; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
490; GFX90A-TGSPLIT-NEXT:    s_endpgm
491;
492; GFX940-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_load:
493; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
494; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
495; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
496; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
497; GFX940-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
498; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
499; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
500; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
501; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
502;
503; GFX940-TGSPLIT-LABEL: local_singlethread_seq_cst_load:
504; GFX940-TGSPLIT:       ; %bb.0: ; %entry
505; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
506; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
507; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
508; GFX940-TGSPLIT-NEXT:    ds_read_b32 v0, v0
509; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
510; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
511; GFX940-TGSPLIT-NEXT:    ds_write_b32 v1, v0
512; GFX940-TGSPLIT-NEXT:    s_endpgm
513;
514; GFX11-WGP-LABEL: local_singlethread_seq_cst_load:
515; GFX11-WGP:       ; %bb.0: ; %entry
516; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
517; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
518; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
519; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
520; GFX11-WGP-NEXT:    ds_load_b32 v0, v0
521; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
522; GFX11-WGP-NEXT:    ds_store_b32 v1, v0
523; GFX11-WGP-NEXT:    s_endpgm
524;
525; GFX11-CU-LABEL: local_singlethread_seq_cst_load:
526; GFX11-CU:       ; %bb.0: ; %entry
527; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
528; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
529; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
530; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
531; GFX11-CU-NEXT:    ds_load_b32 v0, v0
532; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
533; GFX11-CU-NEXT:    ds_store_b32 v1, v0
534; GFX11-CU-NEXT:    s_endpgm
535    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
536entry:
537  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread") seq_cst, align 4
538  store i32 %val, i32 addrspace(3)* %out
539  ret void
540}
541
542define amdgpu_kernel void @local_singlethread_unordered_store(
543; GFX6-LABEL: local_singlethread_unordered_store:
544; GFX6:       ; %bb.0: ; %entry
545; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
546; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
547; GFX6-NEXT:    s_mov_b32 m0, -1
548; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
549; GFX6-NEXT:    v_mov_b32_e32 v0, s0
550; GFX6-NEXT:    v_mov_b32_e32 v1, s1
551; GFX6-NEXT:    ds_write_b32 v0, v1
552; GFX6-NEXT:    s_endpgm
553;
554; GFX7-LABEL: local_singlethread_unordered_store:
555; GFX7:       ; %bb.0: ; %entry
556; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
557; GFX7-NEXT:    s_mov_b32 m0, -1
558; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
559; GFX7-NEXT:    v_mov_b32_e32 v0, s1
560; GFX7-NEXT:    v_mov_b32_e32 v1, s0
561; GFX7-NEXT:    ds_write_b32 v0, v1
562; GFX7-NEXT:    s_endpgm
563;
564; GFX10-WGP-LABEL: local_singlethread_unordered_store:
565; GFX10-WGP:       ; %bb.0: ; %entry
566; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
567; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
568; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
569; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
570; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
571; GFX10-WGP-NEXT:    s_endpgm
572;
573; GFX10-CU-LABEL: local_singlethread_unordered_store:
574; GFX10-CU:       ; %bb.0: ; %entry
575; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
576; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
577; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
578; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
579; GFX10-CU-NEXT:    ds_write_b32 v0, v1
580; GFX10-CU-NEXT:    s_endpgm
581;
582; SKIP-CACHE-INV-LABEL: local_singlethread_unordered_store:
583; SKIP-CACHE-INV:       ; %bb.0: ; %entry
584; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
585; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
586; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
587; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
588; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
589; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
590; SKIP-CACHE-INV-NEXT:    s_endpgm
591;
592; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_unordered_store:
593; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
594; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
595; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
596; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
597; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
598; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
599; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
600;
601; GFX90A-TGSPLIT-LABEL: local_singlethread_unordered_store:
602; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
603; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
604; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
605; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
606; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
607; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
608; GFX90A-TGSPLIT-NEXT:    s_endpgm
609;
610; GFX940-NOTTGSPLIT-LABEL: local_singlethread_unordered_store:
611; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
612; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
613; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
614; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
615; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
616; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
617; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
618;
619; GFX940-TGSPLIT-LABEL: local_singlethread_unordered_store:
620; GFX940-TGSPLIT:       ; %bb.0: ; %entry
621; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
622; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
623; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
624; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
625; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
626; GFX940-TGSPLIT-NEXT:    s_endpgm
627;
628; GFX11-WGP-LABEL: local_singlethread_unordered_store:
629; GFX11-WGP:       ; %bb.0: ; %entry
630; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
631; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
632; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s1
633; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s0
634; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
635; GFX11-WGP-NEXT:    s_endpgm
636;
637; GFX11-CU-LABEL: local_singlethread_unordered_store:
638; GFX11-CU:       ; %bb.0: ; %entry
639; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
640; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
641; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s1
642; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s0
643; GFX11-CU-NEXT:    ds_store_b32 v0, v1
644; GFX11-CU-NEXT:    s_endpgm
645    i32 %in, i32 addrspace(3)* %out) {
646entry:
647  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread") unordered, align 4
648  ret void
649}
650
651define amdgpu_kernel void @local_singlethread_monotonic_store(
652; GFX6-LABEL: local_singlethread_monotonic_store:
653; GFX6:       ; %bb.0: ; %entry
654; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
655; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
656; GFX6-NEXT:    s_mov_b32 m0, -1
657; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
658; GFX6-NEXT:    v_mov_b32_e32 v0, s0
659; GFX6-NEXT:    v_mov_b32_e32 v1, s1
660; GFX6-NEXT:    ds_write_b32 v0, v1
661; GFX6-NEXT:    s_endpgm
662;
663; GFX7-LABEL: local_singlethread_monotonic_store:
664; GFX7:       ; %bb.0: ; %entry
665; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
666; GFX7-NEXT:    s_mov_b32 m0, -1
667; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
668; GFX7-NEXT:    v_mov_b32_e32 v0, s1
669; GFX7-NEXT:    v_mov_b32_e32 v1, s0
670; GFX7-NEXT:    ds_write_b32 v0, v1
671; GFX7-NEXT:    s_endpgm
672;
673; GFX10-WGP-LABEL: local_singlethread_monotonic_store:
674; GFX10-WGP:       ; %bb.0: ; %entry
675; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
676; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
677; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
678; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
679; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
680; GFX10-WGP-NEXT:    s_endpgm
681;
682; GFX10-CU-LABEL: local_singlethread_monotonic_store:
683; GFX10-CU:       ; %bb.0: ; %entry
684; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
685; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
686; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
687; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
688; GFX10-CU-NEXT:    ds_write_b32 v0, v1
689; GFX10-CU-NEXT:    s_endpgm
690;
691; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_store:
692; SKIP-CACHE-INV:       ; %bb.0: ; %entry
693; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
694; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
695; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
696; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
697; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
698; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
699; SKIP-CACHE-INV-NEXT:    s_endpgm
700;
701; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_store:
702; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
703; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
704; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
705; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
706; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
707; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
708; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
709;
710; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_store:
711; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
712; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
713; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
714; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
715; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
716; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
717; GFX90A-TGSPLIT-NEXT:    s_endpgm
718;
719; GFX940-NOTTGSPLIT-LABEL: local_singlethread_monotonic_store:
720; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
721; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
722; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
723; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
724; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
725; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
726; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
727;
728; GFX940-TGSPLIT-LABEL: local_singlethread_monotonic_store:
729; GFX940-TGSPLIT:       ; %bb.0: ; %entry
730; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
731; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
732; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
733; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
734; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
735; GFX940-TGSPLIT-NEXT:    s_endpgm
736;
737; GFX11-WGP-LABEL: local_singlethread_monotonic_store:
738; GFX11-WGP:       ; %bb.0: ; %entry
739; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
740; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
741; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s1
742; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s0
743; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
744; GFX11-WGP-NEXT:    s_endpgm
745;
746; GFX11-CU-LABEL: local_singlethread_monotonic_store:
747; GFX11-CU:       ; %bb.0: ; %entry
748; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
749; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
750; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s1
751; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s0
752; GFX11-CU-NEXT:    ds_store_b32 v0, v1
753; GFX11-CU-NEXT:    s_endpgm
754    i32 %in, i32 addrspace(3)* %out) {
755entry:
756  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread") monotonic, align 4
757  ret void
758}
759
760define amdgpu_kernel void @local_singlethread_release_store(
761; GFX6-LABEL: local_singlethread_release_store:
762; GFX6:       ; %bb.0: ; %entry
763; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
764; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
765; GFX6-NEXT:    s_mov_b32 m0, -1
766; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
767; GFX6-NEXT:    v_mov_b32_e32 v0, s0
768; GFX6-NEXT:    v_mov_b32_e32 v1, s1
769; GFX6-NEXT:    ds_write_b32 v0, v1
770; GFX6-NEXT:    s_endpgm
771;
772; GFX7-LABEL: local_singlethread_release_store:
773; GFX7:       ; %bb.0: ; %entry
774; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
775; GFX7-NEXT:    s_mov_b32 m0, -1
776; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
777; GFX7-NEXT:    v_mov_b32_e32 v0, s1
778; GFX7-NEXT:    v_mov_b32_e32 v1, s0
779; GFX7-NEXT:    ds_write_b32 v0, v1
780; GFX7-NEXT:    s_endpgm
781;
782; GFX10-WGP-LABEL: local_singlethread_release_store:
783; GFX10-WGP:       ; %bb.0: ; %entry
784; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
785; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
786; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
787; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
788; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
789; GFX10-WGP-NEXT:    s_endpgm
790;
791; GFX10-CU-LABEL: local_singlethread_release_store:
792; GFX10-CU:       ; %bb.0: ; %entry
793; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
794; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
795; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
796; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
797; GFX10-CU-NEXT:    ds_write_b32 v0, v1
798; GFX10-CU-NEXT:    s_endpgm
799;
800; SKIP-CACHE-INV-LABEL: local_singlethread_release_store:
801; SKIP-CACHE-INV:       ; %bb.0: ; %entry
802; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
803; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
804; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
805; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
806; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
807; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
808; SKIP-CACHE-INV-NEXT:    s_endpgm
809;
810; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_store:
811; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
812; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
813; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
814; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
815; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
816; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
817; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
818;
819; GFX90A-TGSPLIT-LABEL: local_singlethread_release_store:
820; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
821; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
822; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
823; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
824; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
825; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
826; GFX90A-TGSPLIT-NEXT:    s_endpgm
827;
828; GFX940-NOTTGSPLIT-LABEL: local_singlethread_release_store:
829; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
830; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
831; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
832; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
833; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
834; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
835; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
836;
837; GFX940-TGSPLIT-LABEL: local_singlethread_release_store:
838; GFX940-TGSPLIT:       ; %bb.0: ; %entry
839; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
840; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
841; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
842; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
843; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
844; GFX940-TGSPLIT-NEXT:    s_endpgm
845;
846; GFX11-WGP-LABEL: local_singlethread_release_store:
847; GFX11-WGP:       ; %bb.0: ; %entry
848; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
849; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
850; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s1
851; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s0
852; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
853; GFX11-WGP-NEXT:    s_endpgm
854;
855; GFX11-CU-LABEL: local_singlethread_release_store:
856; GFX11-CU:       ; %bb.0: ; %entry
857; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
858; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
859; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s1
860; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s0
861; GFX11-CU-NEXT:    ds_store_b32 v0, v1
862; GFX11-CU-NEXT:    s_endpgm
863    i32 %in, i32 addrspace(3)* %out) {
864entry:
865  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread") release, align 4
866  ret void
867}
868
869define amdgpu_kernel void @local_singlethread_seq_cst_store(
870; GFX6-LABEL: local_singlethread_seq_cst_store:
871; GFX6:       ; %bb.0: ; %entry
872; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
873; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
874; GFX6-NEXT:    s_mov_b32 m0, -1
875; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
876; GFX6-NEXT:    v_mov_b32_e32 v0, s0
877; GFX6-NEXT:    v_mov_b32_e32 v1, s1
878; GFX6-NEXT:    ds_write_b32 v0, v1
879; GFX6-NEXT:    s_endpgm
880;
881; GFX7-LABEL: local_singlethread_seq_cst_store:
882; GFX7:       ; %bb.0: ; %entry
883; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
884; GFX7-NEXT:    s_mov_b32 m0, -1
885; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
886; GFX7-NEXT:    v_mov_b32_e32 v0, s1
887; GFX7-NEXT:    v_mov_b32_e32 v1, s0
888; GFX7-NEXT:    ds_write_b32 v0, v1
889; GFX7-NEXT:    s_endpgm
890;
891; GFX10-WGP-LABEL: local_singlethread_seq_cst_store:
892; GFX10-WGP:       ; %bb.0: ; %entry
893; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
894; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
895; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
896; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
897; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
898; GFX10-WGP-NEXT:    s_endpgm
899;
900; GFX10-CU-LABEL: local_singlethread_seq_cst_store:
901; GFX10-CU:       ; %bb.0: ; %entry
902; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
903; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
904; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
905; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
906; GFX10-CU-NEXT:    ds_write_b32 v0, v1
907; GFX10-CU-NEXT:    s_endpgm
908;
909; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_store:
910; SKIP-CACHE-INV:       ; %bb.0: ; %entry
911; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
912; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
913; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
914; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
915; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
916; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
917; SKIP-CACHE-INV-NEXT:    s_endpgm
918;
919; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_store:
920; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
921; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
922; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
923; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
924; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
925; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
926; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
927;
928; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_store:
929; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
930; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
931; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
932; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
933; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
934; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
935; GFX90A-TGSPLIT-NEXT:    s_endpgm
936;
937; GFX940-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_store:
938; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
939; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
940; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
941; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
942; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
943; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
944; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
945;
946; GFX940-TGSPLIT-LABEL: local_singlethread_seq_cst_store:
947; GFX940-TGSPLIT:       ; %bb.0: ; %entry
948; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
949; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
950; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
951; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
952; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
953; GFX940-TGSPLIT-NEXT:    s_endpgm
954;
955; GFX11-WGP-LABEL: local_singlethread_seq_cst_store:
956; GFX11-WGP:       ; %bb.0: ; %entry
957; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
958; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
959; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s1
960; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s0
961; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
962; GFX11-WGP-NEXT:    s_endpgm
963;
964; GFX11-CU-LABEL: local_singlethread_seq_cst_store:
965; GFX11-CU:       ; %bb.0: ; %entry
966; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
967; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
968; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s1
969; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s0
970; GFX11-CU-NEXT:    ds_store_b32 v0, v1
971; GFX11-CU-NEXT:    s_endpgm
972    i32 %in, i32 addrspace(3)* %out) {
973entry:
974  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread") seq_cst, align 4
975  ret void
976}
977
978define amdgpu_kernel void @local_singlethread_monotonic_atomicrmw(
979; GFX6-LABEL: local_singlethread_monotonic_atomicrmw:
980; GFX6:       ; %bb.0: ; %entry
981; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
982; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
983; GFX6-NEXT:    s_mov_b32 m0, -1
984; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
985; GFX6-NEXT:    v_mov_b32_e32 v0, s0
986; GFX6-NEXT:    v_mov_b32_e32 v1, s1
987; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
988; GFX6-NEXT:    s_endpgm
989;
990; GFX7-LABEL: local_singlethread_monotonic_atomicrmw:
991; GFX7:       ; %bb.0: ; %entry
992; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
993; GFX7-NEXT:    s_mov_b32 m0, -1
994; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
995; GFX7-NEXT:    v_mov_b32_e32 v0, s0
996; GFX7-NEXT:    v_mov_b32_e32 v1, s1
997; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
998; GFX7-NEXT:    s_endpgm
999;
1000; GFX10-WGP-LABEL: local_singlethread_monotonic_atomicrmw:
1001; GFX10-WGP:       ; %bb.0: ; %entry
1002; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1003; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1004; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1005; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1006; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1007; GFX10-WGP-NEXT:    s_endpgm
1008;
1009; GFX10-CU-LABEL: local_singlethread_monotonic_atomicrmw:
1010; GFX10-CU:       ; %bb.0: ; %entry
1011; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1012; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1013; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1014; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1015; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1016; GFX10-CU-NEXT:    s_endpgm
1017;
1018; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_atomicrmw:
1019; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1020; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1021; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1022; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1023; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1024; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1025; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1026; SKIP-CACHE-INV-NEXT:    s_endpgm
1027;
1028; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_atomicrmw:
1029; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1030; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1031; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1032; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1033; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1034; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1035; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1036;
1037; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_atomicrmw:
1038; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1039; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1040; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1041; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1042; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1043; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1044; GFX90A-TGSPLIT-NEXT:    s_endpgm
1045;
1046; GFX940-NOTTGSPLIT-LABEL: local_singlethread_monotonic_atomicrmw:
1047; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1048; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1049; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1050; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1051; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1052; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1053; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1054;
1055; GFX940-TGSPLIT-LABEL: local_singlethread_monotonic_atomicrmw:
1056; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1057; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1058; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1059; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1060; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1061; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1062; GFX940-TGSPLIT-NEXT:    s_endpgm
1063;
1064; GFX11-WGP-LABEL: local_singlethread_monotonic_atomicrmw:
1065; GFX11-WGP:       ; %bb.0: ; %entry
1066; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1067; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1068; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
1069; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
1070; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1071; GFX11-WGP-NEXT:    s_endpgm
1072;
1073; GFX11-CU-LABEL: local_singlethread_monotonic_atomicrmw:
1074; GFX11-CU:       ; %bb.0: ; %entry
1075; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1076; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1077; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
1078; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
1079; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1080; GFX11-CU-NEXT:    s_endpgm
1081    i32 addrspace(3)* %out, i32 %in) {
1082entry:
1083  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") monotonic
1084  ret void
1085}
1086
1087define amdgpu_kernel void @local_singlethread_acquire_atomicrmw(
1088; GFX6-LABEL: local_singlethread_acquire_atomicrmw:
1089; GFX6:       ; %bb.0: ; %entry
1090; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
1091; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
1092; GFX6-NEXT:    s_mov_b32 m0, -1
1093; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1094; GFX6-NEXT:    v_mov_b32_e32 v0, s0
1095; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1096; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1097; GFX6-NEXT:    s_endpgm
1098;
1099; GFX7-LABEL: local_singlethread_acquire_atomicrmw:
1100; GFX7:       ; %bb.0: ; %entry
1101; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1102; GFX7-NEXT:    s_mov_b32 m0, -1
1103; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1104; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1105; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1106; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1107; GFX7-NEXT:    s_endpgm
1108;
1109; GFX10-WGP-LABEL: local_singlethread_acquire_atomicrmw:
1110; GFX10-WGP:       ; %bb.0: ; %entry
1111; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1112; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1113; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1114; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1115; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1116; GFX10-WGP-NEXT:    s_endpgm
1117;
1118; GFX10-CU-LABEL: local_singlethread_acquire_atomicrmw:
1119; GFX10-CU:       ; %bb.0: ; %entry
1120; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1121; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1122; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1123; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1124; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1125; GFX10-CU-NEXT:    s_endpgm
1126;
1127; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_atomicrmw:
1128; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1129; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1130; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1131; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1132; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1133; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1134; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1135; SKIP-CACHE-INV-NEXT:    s_endpgm
1136;
1137; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_atomicrmw:
1138; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1139; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1140; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1141; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1142; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1143; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1144; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1145;
1146; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_atomicrmw:
1147; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1148; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1149; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1150; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1151; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1152; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1153; GFX90A-TGSPLIT-NEXT:    s_endpgm
1154;
1155; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acquire_atomicrmw:
1156; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1157; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1158; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1159; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1160; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1161; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1162; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1163;
1164; GFX940-TGSPLIT-LABEL: local_singlethread_acquire_atomicrmw:
1165; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1166; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1167; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1168; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1169; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1170; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1171; GFX940-TGSPLIT-NEXT:    s_endpgm
1172;
1173; GFX11-WGP-LABEL: local_singlethread_acquire_atomicrmw:
1174; GFX11-WGP:       ; %bb.0: ; %entry
1175; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1176; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1177; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
1178; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
1179; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1180; GFX11-WGP-NEXT:    s_endpgm
1181;
1182; GFX11-CU-LABEL: local_singlethread_acquire_atomicrmw:
1183; GFX11-CU:       ; %bb.0: ; %entry
1184; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1185; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1186; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
1187; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
1188; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1189; GFX11-CU-NEXT:    s_endpgm
1190    i32 addrspace(3)* %out, i32 %in) {
1191entry:
1192  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") acquire
1193  ret void
1194}
1195
1196define amdgpu_kernel void @local_singlethread_release_atomicrmw(
1197; GFX6-LABEL: local_singlethread_release_atomicrmw:
1198; GFX6:       ; %bb.0: ; %entry
1199; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
1200; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
1201; GFX6-NEXT:    s_mov_b32 m0, -1
1202; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1203; GFX6-NEXT:    v_mov_b32_e32 v0, s0
1204; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1205; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1206; GFX6-NEXT:    s_endpgm
1207;
1208; GFX7-LABEL: local_singlethread_release_atomicrmw:
1209; GFX7:       ; %bb.0: ; %entry
1210; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1211; GFX7-NEXT:    s_mov_b32 m0, -1
1212; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1213; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1214; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1215; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1216; GFX7-NEXT:    s_endpgm
1217;
1218; GFX10-WGP-LABEL: local_singlethread_release_atomicrmw:
1219; GFX10-WGP:       ; %bb.0: ; %entry
1220; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1221; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1222; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1223; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1224; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1225; GFX10-WGP-NEXT:    s_endpgm
1226;
1227; GFX10-CU-LABEL: local_singlethread_release_atomicrmw:
1228; GFX10-CU:       ; %bb.0: ; %entry
1229; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1230; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1231; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1232; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1233; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1234; GFX10-CU-NEXT:    s_endpgm
1235;
1236; SKIP-CACHE-INV-LABEL: local_singlethread_release_atomicrmw:
1237; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1238; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1239; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1240; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1241; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1242; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1243; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1244; SKIP-CACHE-INV-NEXT:    s_endpgm
1245;
1246; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_atomicrmw:
1247; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1248; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1249; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1250; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1251; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1252; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1253; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1254;
1255; GFX90A-TGSPLIT-LABEL: local_singlethread_release_atomicrmw:
1256; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1257; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1258; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1259; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1260; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1261; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1262; GFX90A-TGSPLIT-NEXT:    s_endpgm
1263;
1264; GFX940-NOTTGSPLIT-LABEL: local_singlethread_release_atomicrmw:
1265; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1266; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1267; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1268; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1269; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1270; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1271; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1272;
1273; GFX940-TGSPLIT-LABEL: local_singlethread_release_atomicrmw:
1274; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1275; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1276; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1277; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1278; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1279; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1280; GFX940-TGSPLIT-NEXT:    s_endpgm
1281;
1282; GFX11-WGP-LABEL: local_singlethread_release_atomicrmw:
1283; GFX11-WGP:       ; %bb.0: ; %entry
1284; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1285; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1286; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
1287; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
1288; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1289; GFX11-WGP-NEXT:    s_endpgm
1290;
1291; GFX11-CU-LABEL: local_singlethread_release_atomicrmw:
1292; GFX11-CU:       ; %bb.0: ; %entry
1293; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1294; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1295; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
1296; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
1297; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1298; GFX11-CU-NEXT:    s_endpgm
1299    i32 addrspace(3)* %out, i32 %in) {
1300entry:
1301  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") release
1302  ret void
1303}
1304
1305define amdgpu_kernel void @local_singlethread_acq_rel_atomicrmw(
1306; GFX6-LABEL: local_singlethread_acq_rel_atomicrmw:
1307; GFX6:       ; %bb.0: ; %entry
1308; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
1309; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
1310; GFX6-NEXT:    s_mov_b32 m0, -1
1311; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1312; GFX6-NEXT:    v_mov_b32_e32 v0, s0
1313; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1314; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1315; GFX6-NEXT:    s_endpgm
1316;
1317; GFX7-LABEL: local_singlethread_acq_rel_atomicrmw:
1318; GFX7:       ; %bb.0: ; %entry
1319; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1320; GFX7-NEXT:    s_mov_b32 m0, -1
1321; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1322; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1323; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1324; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1325; GFX7-NEXT:    s_endpgm
1326;
1327; GFX10-WGP-LABEL: local_singlethread_acq_rel_atomicrmw:
1328; GFX10-WGP:       ; %bb.0: ; %entry
1329; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1330; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1331; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1332; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1333; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1334; GFX10-WGP-NEXT:    s_endpgm
1335;
1336; GFX10-CU-LABEL: local_singlethread_acq_rel_atomicrmw:
1337; GFX10-CU:       ; %bb.0: ; %entry
1338; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1339; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1340; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1341; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1342; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1343; GFX10-CU-NEXT:    s_endpgm
1344;
1345; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_atomicrmw:
1346; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1347; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1348; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1349; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1350; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1351; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1352; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1353; SKIP-CACHE-INV-NEXT:    s_endpgm
1354;
1355; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_atomicrmw:
1356; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1357; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1358; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1359; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1360; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1361; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1362; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1363;
1364; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_atomicrmw:
1365; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1366; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1367; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1368; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1369; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1370; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1371; GFX90A-TGSPLIT-NEXT:    s_endpgm
1372;
1373; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_atomicrmw:
1374; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1375; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1376; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1377; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1378; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1379; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1380; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1381;
1382; GFX940-TGSPLIT-LABEL: local_singlethread_acq_rel_atomicrmw:
1383; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1384; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1385; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1386; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1387; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1388; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1389; GFX940-TGSPLIT-NEXT:    s_endpgm
1390;
1391; GFX11-WGP-LABEL: local_singlethread_acq_rel_atomicrmw:
1392; GFX11-WGP:       ; %bb.0: ; %entry
1393; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1394; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1395; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
1396; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
1397; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1398; GFX11-WGP-NEXT:    s_endpgm
1399;
1400; GFX11-CU-LABEL: local_singlethread_acq_rel_atomicrmw:
1401; GFX11-CU:       ; %bb.0: ; %entry
1402; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1403; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1404; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
1405; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
1406; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1407; GFX11-CU-NEXT:    s_endpgm
1408    i32 addrspace(3)* %out, i32 %in) {
1409entry:
1410  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") acq_rel
1411  ret void
1412}
1413
1414define amdgpu_kernel void @local_singlethread_seq_cst_atomicrmw(
1415; GFX6-LABEL: local_singlethread_seq_cst_atomicrmw:
1416; GFX6:       ; %bb.0: ; %entry
1417; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
1418; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
1419; GFX6-NEXT:    s_mov_b32 m0, -1
1420; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1421; GFX6-NEXT:    v_mov_b32_e32 v0, s0
1422; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1423; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1424; GFX6-NEXT:    s_endpgm
1425;
1426; GFX7-LABEL: local_singlethread_seq_cst_atomicrmw:
1427; GFX7:       ; %bb.0: ; %entry
1428; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1429; GFX7-NEXT:    s_mov_b32 m0, -1
1430; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1431; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1432; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1433; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1434; GFX7-NEXT:    s_endpgm
1435;
1436; GFX10-WGP-LABEL: local_singlethread_seq_cst_atomicrmw:
1437; GFX10-WGP:       ; %bb.0: ; %entry
1438; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1439; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1440; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1441; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1442; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1443; GFX10-WGP-NEXT:    s_endpgm
1444;
1445; GFX10-CU-LABEL: local_singlethread_seq_cst_atomicrmw:
1446; GFX10-CU:       ; %bb.0: ; %entry
1447; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1448; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1449; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1450; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1451; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1452; GFX10-CU-NEXT:    s_endpgm
1453;
1454; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_atomicrmw:
1455; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1456; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1457; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1458; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1459; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1460; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1461; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1462; SKIP-CACHE-INV-NEXT:    s_endpgm
1463;
1464; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_atomicrmw:
1465; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1466; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1467; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1468; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1469; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1470; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1471; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1472;
1473; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_atomicrmw:
1474; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1475; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1476; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1477; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1478; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1479; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1480; GFX90A-TGSPLIT-NEXT:    s_endpgm
1481;
1482; GFX940-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_atomicrmw:
1483; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1484; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1485; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1486; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1487; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1488; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1489; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1490;
1491; GFX940-TGSPLIT-LABEL: local_singlethread_seq_cst_atomicrmw:
1492; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1493; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1494; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1495; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1496; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1497; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1498; GFX940-TGSPLIT-NEXT:    s_endpgm
1499;
1500; GFX11-WGP-LABEL: local_singlethread_seq_cst_atomicrmw:
1501; GFX11-WGP:       ; %bb.0: ; %entry
1502; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1503; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1504; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
1505; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
1506; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1507; GFX11-WGP-NEXT:    s_endpgm
1508;
1509; GFX11-CU-LABEL: local_singlethread_seq_cst_atomicrmw:
1510; GFX11-CU:       ; %bb.0: ; %entry
1511; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1512; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1513; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
1514; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
1515; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1516; GFX11-CU-NEXT:    s_endpgm
1517    i32 addrspace(3)* %out, i32 %in) {
1518entry:
1519  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") seq_cst
1520  ret void
1521}
1522
1523define amdgpu_kernel void @local_singlethread_acquire_ret_atomicrmw(
1524; GFX6-LABEL: local_singlethread_acquire_ret_atomicrmw:
1525; GFX6:       ; %bb.0: ; %entry
1526; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
1527; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
1528; GFX6-NEXT:    s_mov_b32 m0, -1
1529; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1530; GFX6-NEXT:    v_mov_b32_e32 v0, s0
1531; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1532; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1533; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1534; GFX6-NEXT:    ds_write_b32 v0, v1
1535; GFX6-NEXT:    s_endpgm
1536;
1537; GFX7-LABEL: local_singlethread_acquire_ret_atomicrmw:
1538; GFX7:       ; %bb.0: ; %entry
1539; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1540; GFX7-NEXT:    s_mov_b32 m0, -1
1541; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1542; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1543; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1544; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1545; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1546; GFX7-NEXT:    ds_write_b32 v0, v1
1547; GFX7-NEXT:    s_endpgm
1548;
1549; GFX10-WGP-LABEL: local_singlethread_acquire_ret_atomicrmw:
1550; GFX10-WGP:       ; %bb.0: ; %entry
1551; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1552; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1553; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1554; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1555; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1556; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1557; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
1558; GFX10-WGP-NEXT:    s_endpgm
1559;
1560; GFX10-CU-LABEL: local_singlethread_acquire_ret_atomicrmw:
1561; GFX10-CU:       ; %bb.0: ; %entry
1562; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1563; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1564; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1565; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1566; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1567; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1568; GFX10-CU-NEXT:    ds_write_b32 v0, v1
1569; GFX10-CU-NEXT:    s_endpgm
1570;
1571; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_ret_atomicrmw:
1572; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1573; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1574; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1575; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1576; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1577; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1578; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1579; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1580; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
1581; SKIP-CACHE-INV-NEXT:    s_endpgm
1582;
1583; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_ret_atomicrmw:
1584; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1585; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1586; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1587; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1588; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1589; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1590; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1591; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
1592; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1593;
1594; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_ret_atomicrmw:
1595; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1596; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1597; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1598; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1599; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1600; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1601; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1602; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
1603; GFX90A-TGSPLIT-NEXT:    s_endpgm
1604;
1605; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acquire_ret_atomicrmw:
1606; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1607; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1608; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1609; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1610; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1611; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1612; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1613; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
1614; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1615;
1616; GFX940-TGSPLIT-LABEL: local_singlethread_acquire_ret_atomicrmw:
1617; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1618; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1619; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1620; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1621; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1622; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1623; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1624; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
1625; GFX940-TGSPLIT-NEXT:    s_endpgm
1626;
1627; GFX11-WGP-LABEL: local_singlethread_acquire_ret_atomicrmw:
1628; GFX11-WGP:       ; %bb.0: ; %entry
1629; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1630; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1631; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
1632; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
1633; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
1634; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1635; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
1636; GFX11-WGP-NEXT:    s_endpgm
1637;
1638; GFX11-CU-LABEL: local_singlethread_acquire_ret_atomicrmw:
1639; GFX11-CU:       ; %bb.0: ; %entry
1640; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1641; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1642; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
1643; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
1644; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
1645; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1646; GFX11-CU-NEXT:    ds_store_b32 v0, v1
1647; GFX11-CU-NEXT:    s_endpgm
1648    i32 addrspace(3)* %out, i32 %in) {
1649entry:
1650  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") acquire
1651  store i32 %val, i32 addrspace(3)* %out, align 4
1652  ret void
1653}
1654
1655define amdgpu_kernel void @local_singlethread_acq_rel_ret_atomicrmw(
1656; GFX6-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1657; GFX6:       ; %bb.0: ; %entry
1658; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
1659; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
1660; GFX6-NEXT:    s_mov_b32 m0, -1
1661; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1662; GFX6-NEXT:    v_mov_b32_e32 v0, s0
1663; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1664; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1665; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1666; GFX6-NEXT:    ds_write_b32 v0, v1
1667; GFX6-NEXT:    s_endpgm
1668;
1669; GFX7-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1670; GFX7:       ; %bb.0: ; %entry
1671; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1672; GFX7-NEXT:    s_mov_b32 m0, -1
1673; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1674; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1675; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1676; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1677; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1678; GFX7-NEXT:    ds_write_b32 v0, v1
1679; GFX7-NEXT:    s_endpgm
1680;
1681; GFX10-WGP-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1682; GFX10-WGP:       ; %bb.0: ; %entry
1683; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1684; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1685; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1686; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1687; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1688; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1689; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
1690; GFX10-WGP-NEXT:    s_endpgm
1691;
1692; GFX10-CU-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1693; GFX10-CU:       ; %bb.0: ; %entry
1694; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1695; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1696; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1697; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1698; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1699; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1700; GFX10-CU-NEXT:    ds_write_b32 v0, v1
1701; GFX10-CU-NEXT:    s_endpgm
1702;
1703; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1704; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1705; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1706; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1707; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1708; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1709; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1710; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1711; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1712; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
1713; SKIP-CACHE-INV-NEXT:    s_endpgm
1714;
1715; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1716; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1717; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1718; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1719; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1720; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1721; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1722; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1723; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
1724; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1725;
1726; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1727; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1728; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1729; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1730; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1731; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1732; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1733; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1734; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
1735; GFX90A-TGSPLIT-NEXT:    s_endpgm
1736;
1737; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1738; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1739; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1740; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1741; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1742; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1743; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1744; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1745; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
1746; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1747;
1748; GFX940-TGSPLIT-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1749; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1750; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1751; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1752; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1753; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1754; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1755; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1756; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
1757; GFX940-TGSPLIT-NEXT:    s_endpgm
1758;
1759; GFX11-WGP-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1760; GFX11-WGP:       ; %bb.0: ; %entry
1761; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1762; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1763; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
1764; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
1765; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
1766; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1767; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
1768; GFX11-WGP-NEXT:    s_endpgm
1769;
1770; GFX11-CU-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1771; GFX11-CU:       ; %bb.0: ; %entry
1772; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1773; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1774; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
1775; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
1776; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
1777; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1778; GFX11-CU-NEXT:    ds_store_b32 v0, v1
1779; GFX11-CU-NEXT:    s_endpgm
1780    i32 addrspace(3)* %out, i32 %in) {
1781entry:
1782  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") acq_rel
1783  store i32 %val, i32 addrspace(3)* %out, align 4
1784  ret void
1785}
1786
1787define amdgpu_kernel void @local_singlethread_seq_cst_ret_atomicrmw(
1788; GFX6-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1789; GFX6:       ; %bb.0: ; %entry
1790; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
1791; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
1792; GFX6-NEXT:    s_mov_b32 m0, -1
1793; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1794; GFX6-NEXT:    v_mov_b32_e32 v0, s0
1795; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1796; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1797; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1798; GFX6-NEXT:    ds_write_b32 v0, v1
1799; GFX6-NEXT:    s_endpgm
1800;
1801; GFX7-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1802; GFX7:       ; %bb.0: ; %entry
1803; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1804; GFX7-NEXT:    s_mov_b32 m0, -1
1805; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1806; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1807; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1808; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1809; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1810; GFX7-NEXT:    ds_write_b32 v0, v1
1811; GFX7-NEXT:    s_endpgm
1812;
1813; GFX10-WGP-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1814; GFX10-WGP:       ; %bb.0: ; %entry
1815; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1816; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1817; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1818; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1819; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1820; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1821; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
1822; GFX10-WGP-NEXT:    s_endpgm
1823;
1824; GFX10-CU-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1825; GFX10-CU:       ; %bb.0: ; %entry
1826; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1827; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1828; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1829; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1830; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1831; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1832; GFX10-CU-NEXT:    ds_write_b32 v0, v1
1833; GFX10-CU-NEXT:    s_endpgm
1834;
1835; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1836; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1837; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1838; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1839; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1840; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1841; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1842; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1843; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1844; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
1845; SKIP-CACHE-INV-NEXT:    s_endpgm
1846;
1847; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1848; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1849; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1850; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1851; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1852; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1853; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1854; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1855; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
1856; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1857;
1858; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1859; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1860; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1861; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1862; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1863; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1864; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1865; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1866; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
1867; GFX90A-TGSPLIT-NEXT:    s_endpgm
1868;
1869; GFX940-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1870; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1871; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1872; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1873; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1874; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1875; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1876; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1877; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
1878; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1879;
1880; GFX940-TGSPLIT-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1881; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1882; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1883; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1884; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1885; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1886; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1887; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1888; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
1889; GFX940-TGSPLIT-NEXT:    s_endpgm
1890;
1891; GFX11-WGP-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1892; GFX11-WGP:       ; %bb.0: ; %entry
1893; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1894; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1895; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
1896; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
1897; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
1898; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1899; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
1900; GFX11-WGP-NEXT:    s_endpgm
1901;
1902; GFX11-CU-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1903; GFX11-CU:       ; %bb.0: ; %entry
1904; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1905; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1906; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
1907; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
1908; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
1909; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1910; GFX11-CU-NEXT:    ds_store_b32 v0, v1
1911; GFX11-CU-NEXT:    s_endpgm
1912    i32 addrspace(3)* %out, i32 %in) {
1913entry:
1914  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") seq_cst
1915  store i32 %val, i32 addrspace(3)* %out, align 4
1916  ret void
1917}
1918
1919define amdgpu_kernel void @local_singlethread_monotonic_monotonic_cmpxchg(
1920; GFX6-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1921; GFX6:       ; %bb.0: ; %entry
1922; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
1923; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
1924; GFX6-NEXT:    s_mov_b32 m0, -1
1925; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1926; GFX6-NEXT:    v_mov_b32_e32 v0, s2
1927; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1928; GFX6-NEXT:    v_mov_b32_e32 v2, s0
1929; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1930; GFX6-NEXT:    s_endpgm
1931;
1932; GFX7-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1933; GFX7:       ; %bb.0: ; %entry
1934; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1935; GFX7-NEXT:    s_mov_b32 m0, -1
1936; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1937; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1938; GFX7-NEXT:    v_mov_b32_e32 v1, s2
1939; GFX7-NEXT:    v_mov_b32_e32 v2, s1
1940; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1941; GFX7-NEXT:    s_endpgm
1942;
1943; GFX10-WGP-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1944; GFX10-WGP:       ; %bb.0: ; %entry
1945; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1946; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1947; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1948; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1949; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
1950; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1951; GFX10-WGP-NEXT:    s_endpgm
1952;
1953; GFX10-CU-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1954; GFX10-CU:       ; %bb.0: ; %entry
1955; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1956; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1957; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1958; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1959; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
1960; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1961; GFX10-CU-NEXT:    s_endpgm
1962;
1963; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1964; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1965; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
1966; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1967; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1968; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1969; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
1970; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
1971; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1972; SKIP-CACHE-INV-NEXT:    s_endpgm
1973;
1974; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1975; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1976; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1977; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1978; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1979; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1980; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1981; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1982; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1983;
1984; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1985; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1986; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1987; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1988; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1989; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1990; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1991; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1992; GFX90A-TGSPLIT-NEXT:    s_endpgm
1993;
1994; GFX940-NOTTGSPLIT-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1995; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1996; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
1997; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1998; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1999; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2000; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2001; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2002; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2003;
2004; GFX940-TGSPLIT-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
2005; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2006; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2007; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2008; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2009; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2010; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2011; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2012; GFX940-TGSPLIT-NEXT:    s_endpgm
2013;
2014; GFX11-WGP-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
2015; GFX11-WGP:       ; %bb.0: ; %entry
2016; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2017; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2018; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
2019; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
2020; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
2021; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2022; GFX11-WGP-NEXT:    s_endpgm
2023;
2024; GFX11-CU-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
2025; GFX11-CU:       ; %bb.0: ; %entry
2026; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2027; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2028; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
2029; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
2030; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2031; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2032; GFX11-CU-NEXT:    s_endpgm
2033    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2034entry:
2035  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2036  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
2037  ret void
2038}
2039
2040define amdgpu_kernel void @local_singlethread_acquire_monotonic_cmpxchg(
2041; GFX6-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2042; GFX6:       ; %bb.0: ; %entry
2043; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2044; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2045; GFX6-NEXT:    s_mov_b32 m0, -1
2046; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2047; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2048; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2049; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2050; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2051; GFX6-NEXT:    s_endpgm
2052;
2053; GFX7-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2054; GFX7:       ; %bb.0: ; %entry
2055; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2056; GFX7-NEXT:    s_mov_b32 m0, -1
2057; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2058; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2059; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2060; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2061; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2062; GFX7-NEXT:    s_endpgm
2063;
2064; GFX10-WGP-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2065; GFX10-WGP:       ; %bb.0: ; %entry
2066; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2067; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2068; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2069; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2070; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2071; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2072; GFX10-WGP-NEXT:    s_endpgm
2073;
2074; GFX10-CU-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2075; GFX10-CU:       ; %bb.0: ; %entry
2076; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2077; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2078; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2079; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2080; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2081; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2082; GFX10-CU-NEXT:    s_endpgm
2083;
2084; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2085; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2086; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2087; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2088; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2089; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2090; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2091; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2092; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2093; SKIP-CACHE-INV-NEXT:    s_endpgm
2094;
2095; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2096; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2097; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2098; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2099; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2100; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2101; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2102; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2103; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2104;
2105; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2106; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2107; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2108; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2109; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2110; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2111; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2112; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2113; GFX90A-TGSPLIT-NEXT:    s_endpgm
2114;
2115; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2116; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2117; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2118; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2119; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2120; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2121; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2122; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2123; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2124;
2125; GFX940-TGSPLIT-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2126; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2127; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2128; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2129; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2130; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2131; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2132; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2133; GFX940-TGSPLIT-NEXT:    s_endpgm
2134;
2135; GFX11-WGP-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2136; GFX11-WGP:       ; %bb.0: ; %entry
2137; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2138; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2139; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
2140; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
2141; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
2142; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2143; GFX11-WGP-NEXT:    s_endpgm
2144;
2145; GFX11-CU-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2146; GFX11-CU:       ; %bb.0: ; %entry
2147; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2148; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2149; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
2150; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
2151; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2152; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2153; GFX11-CU-NEXT:    s_endpgm
2154    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2155entry:
2156  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2157  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
2158  ret void
2159}
2160
2161define amdgpu_kernel void @local_singlethread_release_monotonic_cmpxchg(
2162; GFX6-LABEL: local_singlethread_release_monotonic_cmpxchg:
2163; GFX6:       ; %bb.0: ; %entry
2164; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2165; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2166; GFX6-NEXT:    s_mov_b32 m0, -1
2167; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2168; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2169; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2170; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2171; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2172; GFX6-NEXT:    s_endpgm
2173;
2174; GFX7-LABEL: local_singlethread_release_monotonic_cmpxchg:
2175; GFX7:       ; %bb.0: ; %entry
2176; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2177; GFX7-NEXT:    s_mov_b32 m0, -1
2178; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2179; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2180; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2181; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2182; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2183; GFX7-NEXT:    s_endpgm
2184;
2185; GFX10-WGP-LABEL: local_singlethread_release_monotonic_cmpxchg:
2186; GFX10-WGP:       ; %bb.0: ; %entry
2187; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2188; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2189; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2190; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2191; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2192; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2193; GFX10-WGP-NEXT:    s_endpgm
2194;
2195; GFX10-CU-LABEL: local_singlethread_release_monotonic_cmpxchg:
2196; GFX10-CU:       ; %bb.0: ; %entry
2197; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2198; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2199; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2200; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2201; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2202; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2203; GFX10-CU-NEXT:    s_endpgm
2204;
2205; SKIP-CACHE-INV-LABEL: local_singlethread_release_monotonic_cmpxchg:
2206; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2207; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2208; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2209; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2210; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2211; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2212; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2213; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2214; SKIP-CACHE-INV-NEXT:    s_endpgm
2215;
2216; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_monotonic_cmpxchg:
2217; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2218; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2219; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2220; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2221; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2222; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2223; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2224; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2225;
2226; GFX90A-TGSPLIT-LABEL: local_singlethread_release_monotonic_cmpxchg:
2227; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2228; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2229; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2230; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2231; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2232; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2233; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2234; GFX90A-TGSPLIT-NEXT:    s_endpgm
2235;
2236; GFX940-NOTTGSPLIT-LABEL: local_singlethread_release_monotonic_cmpxchg:
2237; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2238; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2239; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2240; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2241; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2242; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2243; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2244; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2245;
2246; GFX940-TGSPLIT-LABEL: local_singlethread_release_monotonic_cmpxchg:
2247; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2248; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2249; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2250; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2251; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2252; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2253; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2254; GFX940-TGSPLIT-NEXT:    s_endpgm
2255;
2256; GFX11-WGP-LABEL: local_singlethread_release_monotonic_cmpxchg:
2257; GFX11-WGP:       ; %bb.0: ; %entry
2258; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2259; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2260; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
2261; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
2262; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
2263; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2264; GFX11-WGP-NEXT:    s_endpgm
2265;
2266; GFX11-CU-LABEL: local_singlethread_release_monotonic_cmpxchg:
2267; GFX11-CU:       ; %bb.0: ; %entry
2268; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2269; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2270; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
2271; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
2272; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2273; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2274; GFX11-CU-NEXT:    s_endpgm
2275    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2276entry:
2277  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2278  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
2279  ret void
2280}
2281
2282define amdgpu_kernel void @local_singlethread_acq_rel_monotonic_cmpxchg(
2283; GFX6-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2284; GFX6:       ; %bb.0: ; %entry
2285; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2286; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2287; GFX6-NEXT:    s_mov_b32 m0, -1
2288; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2289; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2290; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2291; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2292; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2293; GFX6-NEXT:    s_endpgm
2294;
2295; GFX7-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2296; GFX7:       ; %bb.0: ; %entry
2297; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2298; GFX7-NEXT:    s_mov_b32 m0, -1
2299; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2300; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2301; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2302; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2303; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2304; GFX7-NEXT:    s_endpgm
2305;
2306; GFX10-WGP-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2307; GFX10-WGP:       ; %bb.0: ; %entry
2308; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2309; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2310; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2311; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2312; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2313; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2314; GFX10-WGP-NEXT:    s_endpgm
2315;
2316; GFX10-CU-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2317; GFX10-CU:       ; %bb.0: ; %entry
2318; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2319; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2320; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2321; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2322; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2323; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2324; GFX10-CU-NEXT:    s_endpgm
2325;
2326; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2327; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2328; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2329; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2330; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2331; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2332; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2333; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2334; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2335; SKIP-CACHE-INV-NEXT:    s_endpgm
2336;
2337; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2338; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2339; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2340; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2341; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2342; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2343; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2344; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2345; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2346;
2347; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2348; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2349; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2350; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2351; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2352; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2353; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2354; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2355; GFX90A-TGSPLIT-NEXT:    s_endpgm
2356;
2357; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2358; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2359; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2360; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2361; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2362; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2363; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2364; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2365; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2366;
2367; GFX940-TGSPLIT-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2368; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2369; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2370; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2371; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2372; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2373; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2374; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2375; GFX940-TGSPLIT-NEXT:    s_endpgm
2376;
2377; GFX11-WGP-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2378; GFX11-WGP:       ; %bb.0: ; %entry
2379; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2380; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2381; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
2382; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
2383; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
2384; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2385; GFX11-WGP-NEXT:    s_endpgm
2386;
2387; GFX11-CU-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2388; GFX11-CU:       ; %bb.0: ; %entry
2389; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2390; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2391; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
2392; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
2393; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2394; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2395; GFX11-CU-NEXT:    s_endpgm
2396    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2397entry:
2398  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2399  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
2400  ret void
2401}
2402
2403define amdgpu_kernel void @local_singlethread_seq_cst_monotonic_cmpxchg(
2404; GFX6-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2405; GFX6:       ; %bb.0: ; %entry
2406; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2407; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2408; GFX6-NEXT:    s_mov_b32 m0, -1
2409; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2410; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2411; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2412; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2413; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2414; GFX6-NEXT:    s_endpgm
2415;
2416; GFX7-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2417; GFX7:       ; %bb.0: ; %entry
2418; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2419; GFX7-NEXT:    s_mov_b32 m0, -1
2420; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2421; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2422; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2423; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2424; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2425; GFX7-NEXT:    s_endpgm
2426;
2427; GFX10-WGP-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2428; GFX10-WGP:       ; %bb.0: ; %entry
2429; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2430; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2431; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2432; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2433; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2434; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2435; GFX10-WGP-NEXT:    s_endpgm
2436;
2437; GFX10-CU-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2438; GFX10-CU:       ; %bb.0: ; %entry
2439; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2440; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2441; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2442; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2443; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2444; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2445; GFX10-CU-NEXT:    s_endpgm
2446;
2447; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2448; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2449; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2450; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2451; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2452; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2453; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2454; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2455; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2456; SKIP-CACHE-INV-NEXT:    s_endpgm
2457;
2458; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2459; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2460; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2461; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2462; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2463; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2464; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2465; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2466; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2467;
2468; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2469; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2470; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2471; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2472; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2473; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2474; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2475; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2476; GFX90A-TGSPLIT-NEXT:    s_endpgm
2477;
2478; GFX940-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2479; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2480; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2481; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2482; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2483; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2484; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2485; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2486; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2487;
2488; GFX940-TGSPLIT-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2489; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2490; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2491; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2492; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2493; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2494; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2495; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2496; GFX940-TGSPLIT-NEXT:    s_endpgm
2497;
2498; GFX11-WGP-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2499; GFX11-WGP:       ; %bb.0: ; %entry
2500; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2501; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2502; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
2503; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
2504; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
2505; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2506; GFX11-WGP-NEXT:    s_endpgm
2507;
2508; GFX11-CU-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2509; GFX11-CU:       ; %bb.0: ; %entry
2510; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2511; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2512; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
2513; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
2514; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2515; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2516; GFX11-CU-NEXT:    s_endpgm
2517    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2518entry:
2519  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2520  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
2521  ret void
2522}
2523
2524define amdgpu_kernel void @local_singlethread_monotonic_acquire_cmpxchg(
2525; GFX6-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2526; GFX6:       ; %bb.0: ; %entry
2527; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2528; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2529; GFX6-NEXT:    s_mov_b32 m0, -1
2530; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2531; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2532; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2533; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2534; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2535; GFX6-NEXT:    s_endpgm
2536;
2537; GFX7-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2538; GFX7:       ; %bb.0: ; %entry
2539; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2540; GFX7-NEXT:    s_mov_b32 m0, -1
2541; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2542; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2543; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2544; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2545; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2546; GFX7-NEXT:    s_endpgm
2547;
2548; GFX10-WGP-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2549; GFX10-WGP:       ; %bb.0: ; %entry
2550; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2551; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2552; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2553; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2554; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2555; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2556; GFX10-WGP-NEXT:    s_endpgm
2557;
2558; GFX10-CU-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2559; GFX10-CU:       ; %bb.0: ; %entry
2560; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2561; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2562; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2563; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2564; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2565; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2566; GFX10-CU-NEXT:    s_endpgm
2567;
2568; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2569; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2570; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2571; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2572; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2573; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2574; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2575; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2576; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2577; SKIP-CACHE-INV-NEXT:    s_endpgm
2578;
2579; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2580; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2581; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2582; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2583; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2584; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2585; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2586; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2587; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2588;
2589; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2590; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2591; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2592; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2593; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2594; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2595; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2596; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2597; GFX90A-TGSPLIT-NEXT:    s_endpgm
2598;
2599; GFX940-NOTTGSPLIT-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2600; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2601; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2602; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2603; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2604; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2605; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2606; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2607; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2608;
2609; GFX940-TGSPLIT-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2610; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2611; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2612; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2613; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2614; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2615; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2616; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2617; GFX940-TGSPLIT-NEXT:    s_endpgm
2618;
2619; GFX11-WGP-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2620; GFX11-WGP:       ; %bb.0: ; %entry
2621; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2622; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2623; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
2624; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
2625; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
2626; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2627; GFX11-WGP-NEXT:    s_endpgm
2628;
2629; GFX11-CU-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2630; GFX11-CU:       ; %bb.0: ; %entry
2631; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2632; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2633; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
2634; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
2635; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2636; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2637; GFX11-CU-NEXT:    s_endpgm
2638    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2639entry:
2640  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2641  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic acquire
2642  ret void
2643}
2644
2645define amdgpu_kernel void @local_singlethread_acquire_acquire_cmpxchg(
2646; GFX6-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2647; GFX6:       ; %bb.0: ; %entry
2648; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2649; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2650; GFX6-NEXT:    s_mov_b32 m0, -1
2651; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2652; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2653; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2654; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2655; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2656; GFX6-NEXT:    s_endpgm
2657;
2658; GFX7-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2659; GFX7:       ; %bb.0: ; %entry
2660; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2661; GFX7-NEXT:    s_mov_b32 m0, -1
2662; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2663; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2664; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2665; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2666; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2667; GFX7-NEXT:    s_endpgm
2668;
2669; GFX10-WGP-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2670; GFX10-WGP:       ; %bb.0: ; %entry
2671; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2672; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2673; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2674; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2675; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2676; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2677; GFX10-WGP-NEXT:    s_endpgm
2678;
2679; GFX10-CU-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2680; GFX10-CU:       ; %bb.0: ; %entry
2681; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2682; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2683; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2684; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2685; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2686; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2687; GFX10-CU-NEXT:    s_endpgm
2688;
2689; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2690; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2691; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2692; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2693; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2694; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2695; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2696; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2697; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2698; SKIP-CACHE-INV-NEXT:    s_endpgm
2699;
2700; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2701; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2702; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2703; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2704; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2705; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2706; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2707; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2708; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2709;
2710; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2711; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2712; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2713; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2714; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2715; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2716; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2717; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2718; GFX90A-TGSPLIT-NEXT:    s_endpgm
2719;
2720; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2721; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2722; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2723; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2724; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2725; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2726; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2727; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2728; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2729;
2730; GFX940-TGSPLIT-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2731; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2732; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2733; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2734; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2735; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2736; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2737; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2738; GFX940-TGSPLIT-NEXT:    s_endpgm
2739;
2740; GFX11-WGP-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2741; GFX11-WGP:       ; %bb.0: ; %entry
2742; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2743; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2744; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
2745; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
2746; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
2747; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2748; GFX11-WGP-NEXT:    s_endpgm
2749;
2750; GFX11-CU-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2751; GFX11-CU:       ; %bb.0: ; %entry
2752; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2753; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2754; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
2755; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
2756; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2757; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2758; GFX11-CU-NEXT:    s_endpgm
2759    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2760entry:
2761  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2762  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
2763  ret void
2764}
2765
2766define amdgpu_kernel void @local_singlethread_release_acquire_cmpxchg(
2767; GFX6-LABEL: local_singlethread_release_acquire_cmpxchg:
2768; GFX6:       ; %bb.0: ; %entry
2769; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2770; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2771; GFX6-NEXT:    s_mov_b32 m0, -1
2772; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2773; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2774; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2775; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2776; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2777; GFX6-NEXT:    s_endpgm
2778;
2779; GFX7-LABEL: local_singlethread_release_acquire_cmpxchg:
2780; GFX7:       ; %bb.0: ; %entry
2781; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2782; GFX7-NEXT:    s_mov_b32 m0, -1
2783; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2784; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2785; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2786; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2787; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2788; GFX7-NEXT:    s_endpgm
2789;
2790; GFX10-WGP-LABEL: local_singlethread_release_acquire_cmpxchg:
2791; GFX10-WGP:       ; %bb.0: ; %entry
2792; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2793; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2794; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2795; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2796; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2797; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2798; GFX10-WGP-NEXT:    s_endpgm
2799;
2800; GFX10-CU-LABEL: local_singlethread_release_acquire_cmpxchg:
2801; GFX10-CU:       ; %bb.0: ; %entry
2802; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2803; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2804; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2805; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2806; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2807; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2808; GFX10-CU-NEXT:    s_endpgm
2809;
2810; SKIP-CACHE-INV-LABEL: local_singlethread_release_acquire_cmpxchg:
2811; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2812; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2813; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2814; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2815; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2816; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2817; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2818; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2819; SKIP-CACHE-INV-NEXT:    s_endpgm
2820;
2821; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_acquire_cmpxchg:
2822; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2823; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2824; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2825; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2826; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2827; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2828; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2829; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2830;
2831; GFX90A-TGSPLIT-LABEL: local_singlethread_release_acquire_cmpxchg:
2832; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2833; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2834; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2835; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2836; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2837; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2838; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2839; GFX90A-TGSPLIT-NEXT:    s_endpgm
2840;
2841; GFX940-NOTTGSPLIT-LABEL: local_singlethread_release_acquire_cmpxchg:
2842; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2843; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2844; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2845; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2846; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2847; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2848; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2849; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2850;
2851; GFX940-TGSPLIT-LABEL: local_singlethread_release_acquire_cmpxchg:
2852; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2853; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2854; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2855; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2856; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2857; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2858; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2859; GFX940-TGSPLIT-NEXT:    s_endpgm
2860;
2861; GFX11-WGP-LABEL: local_singlethread_release_acquire_cmpxchg:
2862; GFX11-WGP:       ; %bb.0: ; %entry
2863; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2864; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2865; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
2866; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
2867; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
2868; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2869; GFX11-WGP-NEXT:    s_endpgm
2870;
2871; GFX11-CU-LABEL: local_singlethread_release_acquire_cmpxchg:
2872; GFX11-CU:       ; %bb.0: ; %entry
2873; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2874; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2875; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
2876; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
2877; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2878; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2879; GFX11-CU-NEXT:    s_endpgm
2880    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2881entry:
2882  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2883  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
2884  ret void
2885}
2886
2887define amdgpu_kernel void @local_singlethread_acq_rel_acquire_cmpxchg(
2888; GFX6-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2889; GFX6:       ; %bb.0: ; %entry
2890; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2891; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2892; GFX6-NEXT:    s_mov_b32 m0, -1
2893; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2894; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2895; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2896; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2897; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2898; GFX6-NEXT:    s_endpgm
2899;
2900; GFX7-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2901; GFX7:       ; %bb.0: ; %entry
2902; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2903; GFX7-NEXT:    s_mov_b32 m0, -1
2904; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2905; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2906; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2907; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2908; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2909; GFX7-NEXT:    s_endpgm
2910;
2911; GFX10-WGP-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2912; GFX10-WGP:       ; %bb.0: ; %entry
2913; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2914; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2915; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2916; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2917; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2918; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2919; GFX10-WGP-NEXT:    s_endpgm
2920;
2921; GFX10-CU-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2922; GFX10-CU:       ; %bb.0: ; %entry
2923; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2924; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2925; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2926; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2927; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2928; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2929; GFX10-CU-NEXT:    s_endpgm
2930;
2931; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2932; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2933; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2934; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2935; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2936; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2937; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2938; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2939; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2940; SKIP-CACHE-INV-NEXT:    s_endpgm
2941;
2942; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2943; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2944; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2945; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2946; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2947; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2948; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2949; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2950; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2951;
2952; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2953; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2954; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2955; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2956; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2957; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2958; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2959; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2960; GFX90A-TGSPLIT-NEXT:    s_endpgm
2961;
2962; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2963; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2964; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2965; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2966; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2967; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2968; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2969; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2970; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2971;
2972; GFX940-TGSPLIT-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2973; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2974; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2975; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2976; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2977; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2978; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2979; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2980; GFX940-TGSPLIT-NEXT:    s_endpgm
2981;
2982; GFX11-WGP-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2983; GFX11-WGP:       ; %bb.0: ; %entry
2984; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2985; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2986; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
2987; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
2988; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
2989; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2990; GFX11-WGP-NEXT:    s_endpgm
2991;
2992; GFX11-CU-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2993; GFX11-CU:       ; %bb.0: ; %entry
2994; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2995; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2996; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
2997; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
2998; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2999; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3000; GFX11-CU-NEXT:    s_endpgm
3001    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3002entry:
3003  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3004  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
3005  ret void
3006}
3007
3008define amdgpu_kernel void @local_singlethread_seq_cst_acquire_cmpxchg(
3009; GFX6-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
3010; GFX6:       ; %bb.0: ; %entry
3011; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3012; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3013; GFX6-NEXT:    s_mov_b32 m0, -1
3014; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3015; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3016; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3017; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3018; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3019; GFX6-NEXT:    s_endpgm
3020;
3021; GFX7-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
3022; GFX7:       ; %bb.0: ; %entry
3023; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3024; GFX7-NEXT:    s_mov_b32 m0, -1
3025; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3026; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3027; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3028; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3029; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3030; GFX7-NEXT:    s_endpgm
3031;
3032; GFX10-WGP-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
3033; GFX10-WGP:       ; %bb.0: ; %entry
3034; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3035; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3036; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3037; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3038; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3039; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3040; GFX10-WGP-NEXT:    s_endpgm
3041;
3042; GFX10-CU-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
3043; GFX10-CU:       ; %bb.0: ; %entry
3044; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3045; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3046; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3047; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3048; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3049; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3050; GFX10-CU-NEXT:    s_endpgm
3051;
3052; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
3053; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3054; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3055; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3056; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3057; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3058; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3059; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3060; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3061; SKIP-CACHE-INV-NEXT:    s_endpgm
3062;
3063; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
3064; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3065; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3066; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3067; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3068; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3069; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3070; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3071; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3072;
3073; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
3074; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3075; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3076; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3077; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3078; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3079; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3080; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3081; GFX90A-TGSPLIT-NEXT:    s_endpgm
3082;
3083; GFX940-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
3084; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3085; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3086; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3087; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3088; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3089; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3090; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3091; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3092;
3093; GFX940-TGSPLIT-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
3094; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3095; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3096; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3097; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3098; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3099; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3100; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3101; GFX940-TGSPLIT-NEXT:    s_endpgm
3102;
3103; GFX11-WGP-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
3104; GFX11-WGP:       ; %bb.0: ; %entry
3105; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3106; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3107; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
3108; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
3109; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
3110; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3111; GFX11-WGP-NEXT:    s_endpgm
3112;
3113; GFX11-CU-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
3114; GFX11-CU:       ; %bb.0: ; %entry
3115; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3116; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3117; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
3118; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
3119; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3120; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3121; GFX11-CU-NEXT:    s_endpgm
3122    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3123entry:
3124  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3125  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
3126  ret void
3127}
3128
3129define amdgpu_kernel void @local_singlethread_monotonic_seq_cst_cmpxchg(
3130; GFX6-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3131; GFX6:       ; %bb.0: ; %entry
3132; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3133; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3134; GFX6-NEXT:    s_mov_b32 m0, -1
3135; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3136; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3137; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3138; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3139; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3140; GFX6-NEXT:    s_endpgm
3141;
3142; GFX7-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3143; GFX7:       ; %bb.0: ; %entry
3144; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3145; GFX7-NEXT:    s_mov_b32 m0, -1
3146; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3147; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3148; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3149; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3150; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3151; GFX7-NEXT:    s_endpgm
3152;
3153; GFX10-WGP-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3154; GFX10-WGP:       ; %bb.0: ; %entry
3155; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3156; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3157; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3158; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3159; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3160; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3161; GFX10-WGP-NEXT:    s_endpgm
3162;
3163; GFX10-CU-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3164; GFX10-CU:       ; %bb.0: ; %entry
3165; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3166; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3167; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3168; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3169; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3170; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3171; GFX10-CU-NEXT:    s_endpgm
3172;
3173; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3174; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3175; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3176; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3177; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3178; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3179; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3180; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3181; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3182; SKIP-CACHE-INV-NEXT:    s_endpgm
3183;
3184; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3185; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3186; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3187; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3188; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3189; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3190; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3191; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3192; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3193;
3194; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3195; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3196; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3197; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3198; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3199; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3200; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3201; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3202; GFX90A-TGSPLIT-NEXT:    s_endpgm
3203;
3204; GFX940-NOTTGSPLIT-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3205; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3206; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3207; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3208; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3209; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3210; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3211; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3212; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3213;
3214; GFX940-TGSPLIT-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3215; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3216; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3217; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3218; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3219; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3220; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3221; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3222; GFX940-TGSPLIT-NEXT:    s_endpgm
3223;
3224; GFX11-WGP-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3225; GFX11-WGP:       ; %bb.0: ; %entry
3226; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3227; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3228; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
3229; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
3230; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
3231; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3232; GFX11-WGP-NEXT:    s_endpgm
3233;
3234; GFX11-CU-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3235; GFX11-CU:       ; %bb.0: ; %entry
3236; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3237; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3238; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
3239; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
3240; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3241; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3242; GFX11-CU-NEXT:    s_endpgm
3243    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3244entry:
3245  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3246  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic seq_cst
3247  ret void
3248}
3249
3250define amdgpu_kernel void @local_singlethread_acquire_seq_cst_cmpxchg(
3251; GFX6-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3252; GFX6:       ; %bb.0: ; %entry
3253; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3254; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3255; GFX6-NEXT:    s_mov_b32 m0, -1
3256; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3257; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3258; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3259; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3260; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3261; GFX6-NEXT:    s_endpgm
3262;
3263; GFX7-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3264; GFX7:       ; %bb.0: ; %entry
3265; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3266; GFX7-NEXT:    s_mov_b32 m0, -1
3267; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3268; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3269; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3270; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3271; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3272; GFX7-NEXT:    s_endpgm
3273;
3274; GFX10-WGP-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3275; GFX10-WGP:       ; %bb.0: ; %entry
3276; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3277; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3278; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3279; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3280; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3281; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3282; GFX10-WGP-NEXT:    s_endpgm
3283;
3284; GFX10-CU-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3285; GFX10-CU:       ; %bb.0: ; %entry
3286; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3287; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3288; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3289; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3290; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3291; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3292; GFX10-CU-NEXT:    s_endpgm
3293;
3294; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3295; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3296; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3297; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3298; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3299; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3300; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3301; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3302; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3303; SKIP-CACHE-INV-NEXT:    s_endpgm
3304;
3305; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3306; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3307; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3308; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3309; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3310; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3311; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3312; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3313; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3314;
3315; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3316; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3317; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3318; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3319; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3320; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3321; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3322; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3323; GFX90A-TGSPLIT-NEXT:    s_endpgm
3324;
3325; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3326; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3327; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3328; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3329; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3330; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3331; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3332; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3333; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3334;
3335; GFX940-TGSPLIT-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3336; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3337; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3338; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3339; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3340; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3341; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3342; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3343; GFX940-TGSPLIT-NEXT:    s_endpgm
3344;
3345; GFX11-WGP-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3346; GFX11-WGP:       ; %bb.0: ; %entry
3347; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3348; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3349; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
3350; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
3351; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
3352; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3353; GFX11-WGP-NEXT:    s_endpgm
3354;
3355; GFX11-CU-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3356; GFX11-CU:       ; %bb.0: ; %entry
3357; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3358; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3359; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
3360; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
3361; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3362; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3363; GFX11-CU-NEXT:    s_endpgm
3364    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3365entry:
3366  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3367  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire seq_cst
3368  ret void
3369}
3370
3371define amdgpu_kernel void @local_singlethread_release_seq_cst_cmpxchg(
3372; GFX6-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3373; GFX6:       ; %bb.0: ; %entry
3374; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3375; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3376; GFX6-NEXT:    s_mov_b32 m0, -1
3377; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3378; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3379; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3380; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3381; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3382; GFX6-NEXT:    s_endpgm
3383;
3384; GFX7-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3385; GFX7:       ; %bb.0: ; %entry
3386; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3387; GFX7-NEXT:    s_mov_b32 m0, -1
3388; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3389; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3390; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3391; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3392; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3393; GFX7-NEXT:    s_endpgm
3394;
3395; GFX10-WGP-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3396; GFX10-WGP:       ; %bb.0: ; %entry
3397; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3398; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3399; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3400; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3401; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3402; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3403; GFX10-WGP-NEXT:    s_endpgm
3404;
3405; GFX10-CU-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3406; GFX10-CU:       ; %bb.0: ; %entry
3407; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3408; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3409; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3410; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3411; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3412; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3413; GFX10-CU-NEXT:    s_endpgm
3414;
3415; SKIP-CACHE-INV-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3416; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3417; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3418; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3419; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3420; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3421; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3422; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3423; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3424; SKIP-CACHE-INV-NEXT:    s_endpgm
3425;
3426; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3427; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3428; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3429; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3430; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3431; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3432; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3433; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3434; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3435;
3436; GFX90A-TGSPLIT-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3437; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3438; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3439; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3440; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3441; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3442; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3443; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3444; GFX90A-TGSPLIT-NEXT:    s_endpgm
3445;
3446; GFX940-NOTTGSPLIT-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3447; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3448; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3449; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3450; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3451; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3452; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3453; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3454; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3455;
3456; GFX940-TGSPLIT-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3457; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3458; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3459; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3460; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3461; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3462; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3463; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3464; GFX940-TGSPLIT-NEXT:    s_endpgm
3465;
3466; GFX11-WGP-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3467; GFX11-WGP:       ; %bb.0: ; %entry
3468; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3469; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3470; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
3471; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
3472; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
3473; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3474; GFX11-WGP-NEXT:    s_endpgm
3475;
3476; GFX11-CU-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3477; GFX11-CU:       ; %bb.0: ; %entry
3478; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3479; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3480; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
3481; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
3482; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3483; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3484; GFX11-CU-NEXT:    s_endpgm
3485    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3486entry:
3487  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3488  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release seq_cst
3489  ret void
3490}
3491
3492define amdgpu_kernel void @local_singlethread_acq_rel_seq_cst_cmpxchg(
3493; GFX6-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3494; GFX6:       ; %bb.0: ; %entry
3495; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3496; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3497; GFX6-NEXT:    s_mov_b32 m0, -1
3498; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3499; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3500; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3501; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3502; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3503; GFX6-NEXT:    s_endpgm
3504;
3505; GFX7-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3506; GFX7:       ; %bb.0: ; %entry
3507; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3508; GFX7-NEXT:    s_mov_b32 m0, -1
3509; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3510; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3511; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3512; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3513; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3514; GFX7-NEXT:    s_endpgm
3515;
3516; GFX10-WGP-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3517; GFX10-WGP:       ; %bb.0: ; %entry
3518; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3519; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3520; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3521; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3522; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3523; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3524; GFX10-WGP-NEXT:    s_endpgm
3525;
3526; GFX10-CU-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3527; GFX10-CU:       ; %bb.0: ; %entry
3528; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3529; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3530; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3531; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3532; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3533; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3534; GFX10-CU-NEXT:    s_endpgm
3535;
3536; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3537; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3538; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3539; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3540; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3541; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3542; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3543; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3544; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3545; SKIP-CACHE-INV-NEXT:    s_endpgm
3546;
3547; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3548; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3549; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3550; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3551; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3552; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3553; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3554; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3555; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3556;
3557; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3558; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3559; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3560; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3561; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3562; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3563; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3564; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3565; GFX90A-TGSPLIT-NEXT:    s_endpgm
3566;
3567; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3568; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3569; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3570; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3571; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3572; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3573; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3574; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3575; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3576;
3577; GFX940-TGSPLIT-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3578; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3579; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3580; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3581; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3582; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3583; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3584; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3585; GFX940-TGSPLIT-NEXT:    s_endpgm
3586;
3587; GFX11-WGP-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3588; GFX11-WGP:       ; %bb.0: ; %entry
3589; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3590; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3591; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
3592; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
3593; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
3594; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3595; GFX11-WGP-NEXT:    s_endpgm
3596;
3597; GFX11-CU-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3598; GFX11-CU:       ; %bb.0: ; %entry
3599; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3600; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3601; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
3602; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
3603; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3604; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3605; GFX11-CU-NEXT:    s_endpgm
3606    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3607entry:
3608  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3609  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel seq_cst
3610  ret void
3611}
3612
3613define amdgpu_kernel void @local_singlethread_seq_cst_seq_cst_cmpxchg(
3614; GFX6-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3615; GFX6:       ; %bb.0: ; %entry
3616; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3617; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3618; GFX6-NEXT:    s_mov_b32 m0, -1
3619; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3620; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3621; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3622; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3623; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3624; GFX6-NEXT:    s_endpgm
3625;
3626; GFX7-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3627; GFX7:       ; %bb.0: ; %entry
3628; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3629; GFX7-NEXT:    s_mov_b32 m0, -1
3630; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3631; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3632; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3633; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3634; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3635; GFX7-NEXT:    s_endpgm
3636;
3637; GFX10-WGP-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3638; GFX10-WGP:       ; %bb.0: ; %entry
3639; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3640; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3641; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3642; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3643; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3644; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3645; GFX10-WGP-NEXT:    s_endpgm
3646;
3647; GFX10-CU-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3648; GFX10-CU:       ; %bb.0: ; %entry
3649; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3650; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3651; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3652; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3653; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3654; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3655; GFX10-CU-NEXT:    s_endpgm
3656;
3657; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3658; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3659; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3660; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3661; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3662; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3663; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3664; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3665; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3666; SKIP-CACHE-INV-NEXT:    s_endpgm
3667;
3668; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3669; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3670; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3671; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3672; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3673; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3674; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3675; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3676; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3677;
3678; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3679; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3680; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3681; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3682; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3683; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3684; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3685; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3686; GFX90A-TGSPLIT-NEXT:    s_endpgm
3687;
3688; GFX940-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3689; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3690; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3691; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3692; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3693; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3694; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3695; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3696; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3697;
3698; GFX940-TGSPLIT-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3699; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3700; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3701; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3702; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3703; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3704; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3705; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3706; GFX940-TGSPLIT-NEXT:    s_endpgm
3707;
3708; GFX11-WGP-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3709; GFX11-WGP:       ; %bb.0: ; %entry
3710; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3711; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3712; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
3713; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
3714; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
3715; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3716; GFX11-WGP-NEXT:    s_endpgm
3717;
3718; GFX11-CU-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3719; GFX11-CU:       ; %bb.0: ; %entry
3720; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3721; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3722; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
3723; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
3724; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3725; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3726; GFX11-CU-NEXT:    s_endpgm
3727    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3728entry:
3729  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3730  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
3731  ret void
3732}
3733
3734define amdgpu_kernel void @local_singlethread_monotonic_monotonic_ret_cmpxchg(
3735; GFX6-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3736; GFX6:       ; %bb.0: ; %entry
3737; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3738; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3739; GFX6-NEXT:    s_mov_b32 m0, -1
3740; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3741; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3742; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3743; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3744; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3745; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3746; GFX6-NEXT:    ds_write_b32 v0, v1
3747; GFX6-NEXT:    s_endpgm
3748;
3749; GFX7-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3750; GFX7:       ; %bb.0: ; %entry
3751; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3752; GFX7-NEXT:    s_mov_b32 m0, -1
3753; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3754; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3755; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3756; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3757; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3758; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3759; GFX7-NEXT:    ds_write_b32 v0, v1
3760; GFX7-NEXT:    s_endpgm
3761;
3762; GFX10-WGP-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3763; GFX10-WGP:       ; %bb.0: ; %entry
3764; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3765; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3766; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3767; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3768; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3769; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3770; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3771; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
3772; GFX10-WGP-NEXT:    s_endpgm
3773;
3774; GFX10-CU-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3775; GFX10-CU:       ; %bb.0: ; %entry
3776; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3777; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3778; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3779; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3780; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3781; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3782; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3783; GFX10-CU-NEXT:    ds_write_b32 v0, v1
3784; GFX10-CU-NEXT:    s_endpgm
3785;
3786; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3787; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3788; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3789; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3790; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3791; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3792; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3793; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3794; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3795; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3796; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
3797; SKIP-CACHE-INV-NEXT:    s_endpgm
3798;
3799; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3800; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3801; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3802; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3803; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3804; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3805; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3806; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3807; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3808; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
3809; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3810;
3811; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3812; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3813; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3814; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3815; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3816; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3817; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3818; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3819; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3820; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
3821; GFX90A-TGSPLIT-NEXT:    s_endpgm
3822;
3823; GFX940-NOTTGSPLIT-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3824; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3825; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3826; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3827; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3828; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3829; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3830; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3831; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3832; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
3833; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3834;
3835; GFX940-TGSPLIT-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3836; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3837; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3838; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3839; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3840; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3841; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3842; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3843; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3844; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
3845; GFX940-TGSPLIT-NEXT:    s_endpgm
3846;
3847; GFX11-WGP-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3848; GFX11-WGP:       ; %bb.0: ; %entry
3849; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3850; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3851; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
3852; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
3853; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
3854; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
3855; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3856; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
3857; GFX11-WGP-NEXT:    s_endpgm
3858;
3859; GFX11-CU-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3860; GFX11-CU:       ; %bb.0: ; %entry
3861; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3862; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3863; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
3864; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
3865; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3866; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
3867; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3868; GFX11-CU-NEXT:    ds_store_b32 v0, v1
3869; GFX11-CU-NEXT:    s_endpgm
3870    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3871entry:
3872  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3873  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
3874  %val0 = extractvalue { i32, i1 } %val, 0
3875  store i32 %val0, i32 addrspace(3)* %out, align 4
3876  ret void
3877}
3878
3879define amdgpu_kernel void @local_singlethread_acquire_monotonic_ret_cmpxchg(
3880; GFX6-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3881; GFX6:       ; %bb.0: ; %entry
3882; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3883; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3884; GFX6-NEXT:    s_mov_b32 m0, -1
3885; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3886; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3887; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3888; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3889; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3890; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3891; GFX6-NEXT:    ds_write_b32 v0, v1
3892; GFX6-NEXT:    s_endpgm
3893;
3894; GFX7-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3895; GFX7:       ; %bb.0: ; %entry
3896; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3897; GFX7-NEXT:    s_mov_b32 m0, -1
3898; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3899; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3900; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3901; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3902; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3903; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3904; GFX7-NEXT:    ds_write_b32 v0, v1
3905; GFX7-NEXT:    s_endpgm
3906;
3907; GFX10-WGP-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3908; GFX10-WGP:       ; %bb.0: ; %entry
3909; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3910; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3911; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3912; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3913; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3914; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3915; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3916; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
3917; GFX10-WGP-NEXT:    s_endpgm
3918;
3919; GFX10-CU-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3920; GFX10-CU:       ; %bb.0: ; %entry
3921; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3922; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3923; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3924; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3925; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3926; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3927; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3928; GFX10-CU-NEXT:    ds_write_b32 v0, v1
3929; GFX10-CU-NEXT:    s_endpgm
3930;
3931; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3932; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3933; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3934; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3935; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3936; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3937; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3938; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3939; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3940; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3941; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
3942; SKIP-CACHE-INV-NEXT:    s_endpgm
3943;
3944; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3945; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3946; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3947; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3948; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3949; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3950; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3951; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3952; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3953; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
3954; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3955;
3956; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3957; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3958; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3959; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3960; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3961; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3962; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3963; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3964; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3965; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
3966; GFX90A-TGSPLIT-NEXT:    s_endpgm
3967;
3968; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3969; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3970; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3971; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3972; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3973; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3974; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3975; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3976; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3977; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
3978; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3979;
3980; GFX940-TGSPLIT-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3981; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3982; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3983; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3984; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3985; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3986; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3987; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3988; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3989; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
3990; GFX940-TGSPLIT-NEXT:    s_endpgm
3991;
3992; GFX11-WGP-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3993; GFX11-WGP:       ; %bb.0: ; %entry
3994; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3995; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3996; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
3997; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
3998; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
3999; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4000; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4001; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
4002; GFX11-WGP-NEXT:    s_endpgm
4003;
4004; GFX11-CU-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
4005; GFX11-CU:       ; %bb.0: ; %entry
4006; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4007; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4008; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
4009; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
4010; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4011; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4012; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4013; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4014; GFX11-CU-NEXT:    s_endpgm
4015    i32 addrspace(3)* %out, i32 %in, i32 %old) {
4016entry:
4017  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
4018  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
4019  %val0 = extractvalue { i32, i1 } %val, 0
4020  store i32 %val0, i32 addrspace(3)* %out, align 4
4021  ret void
4022}
4023
4024define amdgpu_kernel void @local_singlethread_release_monotonic_ret_cmpxchg(
4025; GFX6-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
4026; GFX6:       ; %bb.0: ; %entry
4027; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
4028; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
4029; GFX6-NEXT:    s_mov_b32 m0, -1
4030; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4031; GFX6-NEXT:    v_mov_b32_e32 v0, s2
4032; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4033; GFX6-NEXT:    v_mov_b32_e32 v2, s0
4034; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4035; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4036; GFX6-NEXT:    ds_write_b32 v0, v1
4037; GFX6-NEXT:    s_endpgm
4038;
4039; GFX7-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
4040; GFX7:       ; %bb.0: ; %entry
4041; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4042; GFX7-NEXT:    s_mov_b32 m0, -1
4043; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4044; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4045; GFX7-NEXT:    v_mov_b32_e32 v1, s2
4046; GFX7-NEXT:    v_mov_b32_e32 v2, s1
4047; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4048; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4049; GFX7-NEXT:    ds_write_b32 v0, v1
4050; GFX7-NEXT:    s_endpgm
4051;
4052; GFX10-WGP-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
4053; GFX10-WGP:       ; %bb.0: ; %entry
4054; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4055; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4056; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4057; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4058; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
4059; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4060; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4061; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
4062; GFX10-WGP-NEXT:    s_endpgm
4063;
4064; GFX10-CU-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
4065; GFX10-CU:       ; %bb.0: ; %entry
4066; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4067; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4068; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4069; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4070; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4071; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4072; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4073; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4074; GFX10-CU-NEXT:    s_endpgm
4075;
4076; SKIP-CACHE-INV-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
4077; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4078; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4079; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4080; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4081; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4082; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
4083; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
4084; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4085; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4086; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
4087; SKIP-CACHE-INV-NEXT:    s_endpgm
4088;
4089; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
4090; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4091; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4092; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4093; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4094; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4095; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4096; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4097; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4098; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4099; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4100;
4101; GFX90A-TGSPLIT-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
4102; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4103; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4104; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4105; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4106; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4107; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4108; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4109; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4110; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4111; GFX90A-TGSPLIT-NEXT:    s_endpgm
4112;
4113; GFX940-NOTTGSPLIT-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
4114; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4115; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4116; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4117; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4118; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4119; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4120; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4121; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4122; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4123; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4124;
4125; GFX940-TGSPLIT-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
4126; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4127; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4128; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4129; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4130; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4131; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4132; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4133; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4134; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4135; GFX940-TGSPLIT-NEXT:    s_endpgm
4136;
4137; GFX11-WGP-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
4138; GFX11-WGP:       ; %bb.0: ; %entry
4139; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4140; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4141; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
4142; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
4143; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
4144; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4145; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4146; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
4147; GFX11-WGP-NEXT:    s_endpgm
4148;
4149; GFX11-CU-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
4150; GFX11-CU:       ; %bb.0: ; %entry
4151; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4152; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4153; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
4154; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
4155; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4156; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4157; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4158; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4159; GFX11-CU-NEXT:    s_endpgm
4160    i32 addrspace(3)* %out, i32 %in, i32 %old) {
4161entry:
4162  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
4163  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
4164  %val0 = extractvalue { i32, i1 } %val, 0
4165  store i32 %val0, i32 addrspace(3)* %out, align 4
4166  ret void
4167}
4168
4169define amdgpu_kernel void @local_singlethread_acq_rel_monotonic_ret_cmpxchg(
4170; GFX6-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4171; GFX6:       ; %bb.0: ; %entry
4172; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
4173; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
4174; GFX6-NEXT:    s_mov_b32 m0, -1
4175; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4176; GFX6-NEXT:    v_mov_b32_e32 v0, s2
4177; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4178; GFX6-NEXT:    v_mov_b32_e32 v2, s0
4179; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4180; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4181; GFX6-NEXT:    ds_write_b32 v0, v1
4182; GFX6-NEXT:    s_endpgm
4183;
4184; GFX7-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4185; GFX7:       ; %bb.0: ; %entry
4186; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4187; GFX7-NEXT:    s_mov_b32 m0, -1
4188; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4189; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4190; GFX7-NEXT:    v_mov_b32_e32 v1, s2
4191; GFX7-NEXT:    v_mov_b32_e32 v2, s1
4192; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4193; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4194; GFX7-NEXT:    ds_write_b32 v0, v1
4195; GFX7-NEXT:    s_endpgm
4196;
4197; GFX10-WGP-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4198; GFX10-WGP:       ; %bb.0: ; %entry
4199; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4200; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4201; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4202; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4203; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
4204; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4205; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4206; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
4207; GFX10-WGP-NEXT:    s_endpgm
4208;
4209; GFX10-CU-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4210; GFX10-CU:       ; %bb.0: ; %entry
4211; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4212; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4213; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4214; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4215; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4216; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4217; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4218; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4219; GFX10-CU-NEXT:    s_endpgm
4220;
4221; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4222; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4223; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4224; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4225; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4226; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4227; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
4228; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
4229; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4230; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4231; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
4232; SKIP-CACHE-INV-NEXT:    s_endpgm
4233;
4234; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4235; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4236; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4237; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4238; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4239; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4240; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4241; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4242; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4243; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4244; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4245;
4246; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4247; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4248; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4249; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4250; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4251; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4252; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4253; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4254; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4255; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4256; GFX90A-TGSPLIT-NEXT:    s_endpgm
4257;
4258; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4259; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4260; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4261; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4262; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4263; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4264; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4265; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4266; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4267; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4268; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4269;
4270; GFX940-TGSPLIT-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4271; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4272; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4273; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4274; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4275; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4276; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4277; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4278; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4279; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4280; GFX940-TGSPLIT-NEXT:    s_endpgm
4281;
4282; GFX11-WGP-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4283; GFX11-WGP:       ; %bb.0: ; %entry
4284; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4285; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4286; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
4287; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
4288; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
4289; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4290; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4291; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
4292; GFX11-WGP-NEXT:    s_endpgm
4293;
4294; GFX11-CU-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4295; GFX11-CU:       ; %bb.0: ; %entry
4296; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4297; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4298; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
4299; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
4300; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4301; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4302; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4303; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4304; GFX11-CU-NEXT:    s_endpgm
4305    i32 addrspace(3)* %out, i32 %in, i32 %old) {
4306entry:
4307  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
4308  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
4309  %val0 = extractvalue { i32, i1 } %val, 0
4310  store i32 %val0, i32 addrspace(3)* %out, align 4
4311  ret void
4312}
4313
4314define amdgpu_kernel void @local_singlethread_seq_cst_monotonic_ret_cmpxchg(
4315; GFX6-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4316; GFX6:       ; %bb.0: ; %entry
4317; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
4318; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
4319; GFX6-NEXT:    s_mov_b32 m0, -1
4320; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4321; GFX6-NEXT:    v_mov_b32_e32 v0, s2
4322; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4323; GFX6-NEXT:    v_mov_b32_e32 v2, s0
4324; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4325; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4326; GFX6-NEXT:    ds_write_b32 v0, v1
4327; GFX6-NEXT:    s_endpgm
4328;
4329; GFX7-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4330; GFX7:       ; %bb.0: ; %entry
4331; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4332; GFX7-NEXT:    s_mov_b32 m0, -1
4333; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4334; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4335; GFX7-NEXT:    v_mov_b32_e32 v1, s2
4336; GFX7-NEXT:    v_mov_b32_e32 v2, s1
4337; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4338; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4339; GFX7-NEXT:    ds_write_b32 v0, v1
4340; GFX7-NEXT:    s_endpgm
4341;
4342; GFX10-WGP-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4343; GFX10-WGP:       ; %bb.0: ; %entry
4344; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4345; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4346; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4347; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4348; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
4349; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4350; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4351; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
4352; GFX10-WGP-NEXT:    s_endpgm
4353;
4354; GFX10-CU-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4355; GFX10-CU:       ; %bb.0: ; %entry
4356; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4357; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4358; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4359; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4360; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4361; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4362; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4363; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4364; GFX10-CU-NEXT:    s_endpgm
4365;
4366; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4367; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4368; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4369; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4370; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4371; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4372; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
4373; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
4374; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4375; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4376; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
4377; SKIP-CACHE-INV-NEXT:    s_endpgm
4378;
4379; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4380; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4381; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4382; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4383; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4384; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4385; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4386; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4387; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4388; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4389; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4390;
4391; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4392; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4393; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4394; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4395; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4396; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4397; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4398; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4399; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4400; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4401; GFX90A-TGSPLIT-NEXT:    s_endpgm
4402;
4403; GFX940-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4404; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4405; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4406; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4407; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4408; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4409; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4410; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4411; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4412; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4413; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4414;
4415; GFX940-TGSPLIT-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4416; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4417; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4418; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4419; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4420; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4421; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4422; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4423; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4424; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4425; GFX940-TGSPLIT-NEXT:    s_endpgm
4426;
4427; GFX11-WGP-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4428; GFX11-WGP:       ; %bb.0: ; %entry
4429; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4430; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4431; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
4432; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
4433; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
4434; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4435; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4436; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
4437; GFX11-WGP-NEXT:    s_endpgm
4438;
4439; GFX11-CU-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4440; GFX11-CU:       ; %bb.0: ; %entry
4441; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4442; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4443; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
4444; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
4445; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4446; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4447; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4448; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4449; GFX11-CU-NEXT:    s_endpgm
4450    i32 addrspace(3)* %out, i32 %in, i32 %old) {
4451entry:
4452  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
4453  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
4454  %val0 = extractvalue { i32, i1 } %val, 0
4455  store i32 %val0, i32 addrspace(3)* %out, align 4
4456  ret void
4457}
4458
4459define amdgpu_kernel void @local_singlethread_monotonic_acquire_ret_cmpxchg(
4460; GFX6-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4461; GFX6:       ; %bb.0: ; %entry
4462; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
4463; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
4464; GFX6-NEXT:    s_mov_b32 m0, -1
4465; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4466; GFX6-NEXT:    v_mov_b32_e32 v0, s2
4467; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4468; GFX6-NEXT:    v_mov_b32_e32 v2, s0
4469; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4470; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4471; GFX6-NEXT:    ds_write_b32 v0, v1
4472; GFX6-NEXT:    s_endpgm
4473;
4474; GFX7-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4475; GFX7:       ; %bb.0: ; %entry
4476; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4477; GFX7-NEXT:    s_mov_b32 m0, -1
4478; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4479; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4480; GFX7-NEXT:    v_mov_b32_e32 v1, s2
4481; GFX7-NEXT:    v_mov_b32_e32 v2, s1
4482; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4483; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4484; GFX7-NEXT:    ds_write_b32 v0, v1
4485; GFX7-NEXT:    s_endpgm
4486;
4487; GFX10-WGP-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4488; GFX10-WGP:       ; %bb.0: ; %entry
4489; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4490; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4491; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4492; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4493; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
4494; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4495; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4496; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
4497; GFX10-WGP-NEXT:    s_endpgm
4498;
4499; GFX10-CU-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4500; GFX10-CU:       ; %bb.0: ; %entry
4501; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4502; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4503; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4504; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4505; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4506; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4507; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4508; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4509; GFX10-CU-NEXT:    s_endpgm
4510;
4511; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4512; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4513; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4514; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4515; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4516; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4517; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
4518; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
4519; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4520; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4521; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
4522; SKIP-CACHE-INV-NEXT:    s_endpgm
4523;
4524; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4525; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4526; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4527; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4528; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4529; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4530; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4531; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4532; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4533; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4534; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4535;
4536; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4537; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4538; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4539; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4540; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4541; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4542; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4543; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4544; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4545; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4546; GFX90A-TGSPLIT-NEXT:    s_endpgm
4547;
4548; GFX940-NOTTGSPLIT-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4549; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4550; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4551; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4552; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4553; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4554; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4555; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4556; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4557; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4558; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4559;
4560; GFX940-TGSPLIT-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4561; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4562; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4563; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4564; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4565; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4566; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4567; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4568; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4569; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4570; GFX940-TGSPLIT-NEXT:    s_endpgm
4571;
4572; GFX11-WGP-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4573; GFX11-WGP:       ; %bb.0: ; %entry
4574; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4575; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4576; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
4577; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
4578; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
4579; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4580; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4581; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
4582; GFX11-WGP-NEXT:    s_endpgm
4583;
4584; GFX11-CU-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4585; GFX11-CU:       ; %bb.0: ; %entry
4586; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4587; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4588; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
4589; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
4590; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4591; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4592; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4593; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4594; GFX11-CU-NEXT:    s_endpgm
4595    i32 addrspace(3)* %out, i32 %in, i32 %old) {
4596entry:
4597  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
4598  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic acquire
4599  %val0 = extractvalue { i32, i1 } %val, 0
4600  store i32 %val0, i32 addrspace(3)* %out, align 4
4601  ret void
4602}
4603
4604define amdgpu_kernel void @local_singlethread_acquire_acquire_ret_cmpxchg(
4605; GFX6-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4606; GFX6:       ; %bb.0: ; %entry
4607; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
4608; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
4609; GFX6-NEXT:    s_mov_b32 m0, -1
4610; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4611; GFX6-NEXT:    v_mov_b32_e32 v0, s2
4612; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4613; GFX6-NEXT:    v_mov_b32_e32 v2, s0
4614; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4615; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4616; GFX6-NEXT:    ds_write_b32 v0, v1
4617; GFX6-NEXT:    s_endpgm
4618;
4619; GFX7-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4620; GFX7:       ; %bb.0: ; %entry
4621; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4622; GFX7-NEXT:    s_mov_b32 m0, -1
4623; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4624; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4625; GFX7-NEXT:    v_mov_b32_e32 v1, s2
4626; GFX7-NEXT:    v_mov_b32_e32 v2, s1
4627; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4628; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4629; GFX7-NEXT:    ds_write_b32 v0, v1
4630; GFX7-NEXT:    s_endpgm
4631;
4632; GFX10-WGP-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4633; GFX10-WGP:       ; %bb.0: ; %entry
4634; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4635; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4636; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4637; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4638; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
4639; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4640; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4641; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
4642; GFX10-WGP-NEXT:    s_endpgm
4643;
4644; GFX10-CU-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4645; GFX10-CU:       ; %bb.0: ; %entry
4646; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4647; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4648; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4649; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4650; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4651; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4652; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4653; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4654; GFX10-CU-NEXT:    s_endpgm
4655;
4656; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4657; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4658; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4659; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4660; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4661; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4662; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
4663; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
4664; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4665; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4666; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
4667; SKIP-CACHE-INV-NEXT:    s_endpgm
4668;
4669; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4670; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4671; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4672; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4673; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4674; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4675; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4676; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4677; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4678; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4679; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4680;
4681; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4682; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4683; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4684; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4685; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4686; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4687; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4688; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4689; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4690; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4691; GFX90A-TGSPLIT-NEXT:    s_endpgm
4692;
4693; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4694; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4695; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4696; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4697; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4698; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4699; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4700; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4701; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4702; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4703; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4704;
4705; GFX940-TGSPLIT-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4706; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4707; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4708; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4709; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4710; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4711; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4712; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4713; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4714; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4715; GFX940-TGSPLIT-NEXT:    s_endpgm
4716;
4717; GFX11-WGP-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4718; GFX11-WGP:       ; %bb.0: ; %entry
4719; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4720; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4721; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
4722; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
4723; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
4724; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4725; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4726; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
4727; GFX11-WGP-NEXT:    s_endpgm
4728;
4729; GFX11-CU-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4730; GFX11-CU:       ; %bb.0: ; %entry
4731; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4732; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4733; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
4734; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
4735; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4736; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4737; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4738; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4739; GFX11-CU-NEXT:    s_endpgm
4740    i32 addrspace(3)* %out, i32 %in, i32 %old) {
4741entry:
4742  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
4743  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
4744  %val0 = extractvalue { i32, i1 } %val, 0
4745  store i32 %val0, i32 addrspace(3)* %out, align 4
4746  ret void
4747}
4748
4749define amdgpu_kernel void @local_singlethread_release_acquire_ret_cmpxchg(
4750; GFX6-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4751; GFX6:       ; %bb.0: ; %entry
4752; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
4753; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
4754; GFX6-NEXT:    s_mov_b32 m0, -1
4755; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4756; GFX6-NEXT:    v_mov_b32_e32 v0, s2
4757; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4758; GFX6-NEXT:    v_mov_b32_e32 v2, s0
4759; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4760; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4761; GFX6-NEXT:    ds_write_b32 v0, v1
4762; GFX6-NEXT:    s_endpgm
4763;
4764; GFX7-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4765; GFX7:       ; %bb.0: ; %entry
4766; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4767; GFX7-NEXT:    s_mov_b32 m0, -1
4768; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4769; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4770; GFX7-NEXT:    v_mov_b32_e32 v1, s2
4771; GFX7-NEXT:    v_mov_b32_e32 v2, s1
4772; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4773; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4774; GFX7-NEXT:    ds_write_b32 v0, v1
4775; GFX7-NEXT:    s_endpgm
4776;
4777; GFX10-WGP-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4778; GFX10-WGP:       ; %bb.0: ; %entry
4779; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4780; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4781; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4782; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4783; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
4784; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4785; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4786; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
4787; GFX10-WGP-NEXT:    s_endpgm
4788;
4789; GFX10-CU-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4790; GFX10-CU:       ; %bb.0: ; %entry
4791; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4792; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4793; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4794; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4795; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4796; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4797; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4798; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4799; GFX10-CU-NEXT:    s_endpgm
4800;
4801; SKIP-CACHE-INV-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4802; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4803; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4804; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4805; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4806; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4807; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
4808; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
4809; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4810; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4811; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
4812; SKIP-CACHE-INV-NEXT:    s_endpgm
4813;
4814; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4815; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4816; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4817; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4818; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4819; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4820; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4821; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4822; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4823; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4824; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4825;
4826; GFX90A-TGSPLIT-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4827; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4828; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4829; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4830; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4831; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4832; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4833; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4834; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4835; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4836; GFX90A-TGSPLIT-NEXT:    s_endpgm
4837;
4838; GFX940-NOTTGSPLIT-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4839; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4840; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4841; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4842; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4843; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4844; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4845; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4846; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4847; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4848; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4849;
4850; GFX940-TGSPLIT-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4851; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4852; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4853; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4854; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4855; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4856; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4857; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4858; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4859; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4860; GFX940-TGSPLIT-NEXT:    s_endpgm
4861;
4862; GFX11-WGP-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4863; GFX11-WGP:       ; %bb.0: ; %entry
4864; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4865; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4866; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
4867; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
4868; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
4869; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4870; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4871; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
4872; GFX11-WGP-NEXT:    s_endpgm
4873;
4874; GFX11-CU-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4875; GFX11-CU:       ; %bb.0: ; %entry
4876; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4877; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4878; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
4879; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
4880; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4881; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4882; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4883; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4884; GFX11-CU-NEXT:    s_endpgm
4885    i32 addrspace(3)* %out, i32 %in, i32 %old) {
4886entry:
4887  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
4888  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
4889  %val0 = extractvalue { i32, i1 } %val, 0
4890  store i32 %val0, i32 addrspace(3)* %out, align 4
4891  ret void
4892}
4893
4894define amdgpu_kernel void @local_singlethread_acq_rel_acquire_ret_cmpxchg(
4895; GFX6-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4896; GFX6:       ; %bb.0: ; %entry
4897; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
4898; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
4899; GFX6-NEXT:    s_mov_b32 m0, -1
4900; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4901; GFX6-NEXT:    v_mov_b32_e32 v0, s2
4902; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4903; GFX6-NEXT:    v_mov_b32_e32 v2, s0
4904; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4905; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4906; GFX6-NEXT:    ds_write_b32 v0, v1
4907; GFX6-NEXT:    s_endpgm
4908;
4909; GFX7-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4910; GFX7:       ; %bb.0: ; %entry
4911; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4912; GFX7-NEXT:    s_mov_b32 m0, -1
4913; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4914; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4915; GFX7-NEXT:    v_mov_b32_e32 v1, s2
4916; GFX7-NEXT:    v_mov_b32_e32 v2, s1
4917; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4918; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4919; GFX7-NEXT:    ds_write_b32 v0, v1
4920; GFX7-NEXT:    s_endpgm
4921;
4922; GFX10-WGP-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4923; GFX10-WGP:       ; %bb.0: ; %entry
4924; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4925; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4926; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4927; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4928; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
4929; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4930; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4931; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
4932; GFX10-WGP-NEXT:    s_endpgm
4933;
4934; GFX10-CU-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4935; GFX10-CU:       ; %bb.0: ; %entry
4936; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4937; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4938; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4939; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4940; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4941; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4942; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4943; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4944; GFX10-CU-NEXT:    s_endpgm
4945;
4946; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4947; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4948; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4949; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4950; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4951; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4952; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
4953; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
4954; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4955; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4956; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
4957; SKIP-CACHE-INV-NEXT:    s_endpgm
4958;
4959; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4960; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4961; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4962; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4963; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4964; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4965; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4966; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4967; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4968; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4969; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4970;
4971; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4972; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4973; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4974; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4975; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4976; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4977; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4978; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4979; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4980; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4981; GFX90A-TGSPLIT-NEXT:    s_endpgm
4982;
4983; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4984; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4985; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4986; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4987; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4988; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4989; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4990; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4991; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4992; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4993; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4994;
4995; GFX940-TGSPLIT-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4996; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4997; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4998; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4999; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5000; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5001; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5002; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5003; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5004; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5005; GFX940-TGSPLIT-NEXT:    s_endpgm
5006;
5007; GFX11-WGP-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
5008; GFX11-WGP:       ; %bb.0: ; %entry
5009; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5010; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5011; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
5012; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
5013; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
5014; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5015; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5016; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
5017; GFX11-WGP-NEXT:    s_endpgm
5018;
5019; GFX11-CU-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
5020; GFX11-CU:       ; %bb.0: ; %entry
5021; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5022; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5023; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
5024; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
5025; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
5026; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5027; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5028; GFX11-CU-NEXT:    ds_store_b32 v0, v1
5029; GFX11-CU-NEXT:    s_endpgm
5030    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5031entry:
5032  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5033  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
5034  %val0 = extractvalue { i32, i1 } %val, 0
5035  store i32 %val0, i32 addrspace(3)* %out, align 4
5036  ret void
5037}
5038
5039define amdgpu_kernel void @local_singlethread_seq_cst_acquire_ret_cmpxchg(
5040; GFX6-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5041; GFX6:       ; %bb.0: ; %entry
5042; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5043; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5044; GFX6-NEXT:    s_mov_b32 m0, -1
5045; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5046; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5047; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5048; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5049; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5050; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5051; GFX6-NEXT:    ds_write_b32 v0, v1
5052; GFX6-NEXT:    s_endpgm
5053;
5054; GFX7-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5055; GFX7:       ; %bb.0: ; %entry
5056; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5057; GFX7-NEXT:    s_mov_b32 m0, -1
5058; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5059; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5060; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5061; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5062; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5063; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5064; GFX7-NEXT:    ds_write_b32 v0, v1
5065; GFX7-NEXT:    s_endpgm
5066;
5067; GFX10-WGP-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5068; GFX10-WGP:       ; %bb.0: ; %entry
5069; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5070; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5071; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5072; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5073; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5074; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5075; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5076; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
5077; GFX10-WGP-NEXT:    s_endpgm
5078;
5079; GFX10-CU-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5080; GFX10-CU:       ; %bb.0: ; %entry
5081; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5082; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5083; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5084; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5085; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5086; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5087; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5088; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5089; GFX10-CU-NEXT:    s_endpgm
5090;
5091; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5092; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5093; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5094; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5095; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5096; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5097; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5098; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5099; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5100; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5101; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
5102; SKIP-CACHE-INV-NEXT:    s_endpgm
5103;
5104; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5105; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5106; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5107; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5108; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5109; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5110; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5111; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5112; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5113; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5114; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5115;
5116; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5117; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5118; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5119; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5120; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5121; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5122; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5123; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5124; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5125; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5126; GFX90A-TGSPLIT-NEXT:    s_endpgm
5127;
5128; GFX940-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5129; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5130; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5131; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5132; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5133; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5134; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5135; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5136; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5137; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5138; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5139;
5140; GFX940-TGSPLIT-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5141; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5142; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5143; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5144; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5145; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5146; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5147; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5148; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5149; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5150; GFX940-TGSPLIT-NEXT:    s_endpgm
5151;
5152; GFX11-WGP-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5153; GFX11-WGP:       ; %bb.0: ; %entry
5154; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5155; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5156; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
5157; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
5158; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
5159; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5160; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5161; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
5162; GFX11-WGP-NEXT:    s_endpgm
5163;
5164; GFX11-CU-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5165; GFX11-CU:       ; %bb.0: ; %entry
5166; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5167; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5168; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
5169; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
5170; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
5171; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5172; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5173; GFX11-CU-NEXT:    ds_store_b32 v0, v1
5174; GFX11-CU-NEXT:    s_endpgm
5175    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5176entry:
5177  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5178  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
5179  %val0 = extractvalue { i32, i1 } %val, 0
5180  store i32 %val0, i32 addrspace(3)* %out, align 4
5181  ret void
5182}
5183
5184define amdgpu_kernel void @local_singlethread_monotonic_seq_cst_ret_cmpxchg(
5185; GFX6-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5186; GFX6:       ; %bb.0: ; %entry
5187; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5188; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5189; GFX6-NEXT:    s_mov_b32 m0, -1
5190; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5191; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5192; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5193; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5194; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5195; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5196; GFX6-NEXT:    ds_write_b32 v0, v1
5197; GFX6-NEXT:    s_endpgm
5198;
5199; GFX7-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5200; GFX7:       ; %bb.0: ; %entry
5201; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5202; GFX7-NEXT:    s_mov_b32 m0, -1
5203; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5204; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5205; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5206; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5207; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5208; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5209; GFX7-NEXT:    ds_write_b32 v0, v1
5210; GFX7-NEXT:    s_endpgm
5211;
5212; GFX10-WGP-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5213; GFX10-WGP:       ; %bb.0: ; %entry
5214; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5215; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5216; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5217; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5218; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5219; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5220; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5221; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
5222; GFX10-WGP-NEXT:    s_endpgm
5223;
5224; GFX10-CU-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5225; GFX10-CU:       ; %bb.0: ; %entry
5226; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5227; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5228; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5229; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5230; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5231; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5232; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5233; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5234; GFX10-CU-NEXT:    s_endpgm
5235;
5236; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5237; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5238; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5239; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5240; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5241; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5242; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5243; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5244; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5245; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5246; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
5247; SKIP-CACHE-INV-NEXT:    s_endpgm
5248;
5249; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5250; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5251; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5252; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5253; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5254; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5255; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5256; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5257; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5258; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5259; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5260;
5261; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5262; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5263; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5264; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5265; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5266; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5267; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5268; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5269; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5270; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5271; GFX90A-TGSPLIT-NEXT:    s_endpgm
5272;
5273; GFX940-NOTTGSPLIT-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5274; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5275; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5276; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5277; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5278; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5279; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5280; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5281; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5282; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5283; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5284;
5285; GFX940-TGSPLIT-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5286; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5287; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5288; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5289; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5290; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5291; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5292; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5293; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5294; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5295; GFX940-TGSPLIT-NEXT:    s_endpgm
5296;
5297; GFX11-WGP-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5298; GFX11-WGP:       ; %bb.0: ; %entry
5299; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5300; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5301; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
5302; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
5303; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
5304; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5305; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5306; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
5307; GFX11-WGP-NEXT:    s_endpgm
5308;
5309; GFX11-CU-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5310; GFX11-CU:       ; %bb.0: ; %entry
5311; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5312; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5313; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
5314; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
5315; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
5316; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5317; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5318; GFX11-CU-NEXT:    ds_store_b32 v0, v1
5319; GFX11-CU-NEXT:    s_endpgm
5320    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5321entry:
5322  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5323  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic seq_cst
5324  %val0 = extractvalue { i32, i1 } %val, 0
5325  store i32 %val0, i32 addrspace(3)* %out, align 4
5326  ret void
5327}
5328
5329define amdgpu_kernel void @local_singlethread_acquire_seq_cst_ret_cmpxchg(
5330; GFX6-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5331; GFX6:       ; %bb.0: ; %entry
5332; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5333; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5334; GFX6-NEXT:    s_mov_b32 m0, -1
5335; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5336; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5337; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5338; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5339; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5340; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5341; GFX6-NEXT:    ds_write_b32 v0, v1
5342; GFX6-NEXT:    s_endpgm
5343;
5344; GFX7-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5345; GFX7:       ; %bb.0: ; %entry
5346; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5347; GFX7-NEXT:    s_mov_b32 m0, -1
5348; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5349; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5350; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5351; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5352; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5353; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5354; GFX7-NEXT:    ds_write_b32 v0, v1
5355; GFX7-NEXT:    s_endpgm
5356;
5357; GFX10-WGP-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5358; GFX10-WGP:       ; %bb.0: ; %entry
5359; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5360; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5361; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5362; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5363; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5364; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5365; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5366; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
5367; GFX10-WGP-NEXT:    s_endpgm
5368;
5369; GFX10-CU-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5370; GFX10-CU:       ; %bb.0: ; %entry
5371; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5372; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5373; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5374; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5375; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5376; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5377; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5378; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5379; GFX10-CU-NEXT:    s_endpgm
5380;
5381; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5382; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5383; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5384; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5385; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5386; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5387; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5388; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5389; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5390; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5391; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
5392; SKIP-CACHE-INV-NEXT:    s_endpgm
5393;
5394; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5395; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5396; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5397; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5398; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5399; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5400; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5401; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5402; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5403; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5404; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5405;
5406; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5407; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5408; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5409; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5410; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5411; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5412; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5413; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5414; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5415; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5416; GFX90A-TGSPLIT-NEXT:    s_endpgm
5417;
5418; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5419; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5420; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5421; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5422; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5423; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5424; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5425; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5426; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5427; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5428; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5429;
5430; GFX940-TGSPLIT-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5431; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5432; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5433; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5434; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5435; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5436; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5437; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5438; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5439; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5440; GFX940-TGSPLIT-NEXT:    s_endpgm
5441;
5442; GFX11-WGP-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5443; GFX11-WGP:       ; %bb.0: ; %entry
5444; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5445; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5446; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
5447; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
5448; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
5449; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5450; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5451; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
5452; GFX11-WGP-NEXT:    s_endpgm
5453;
5454; GFX11-CU-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5455; GFX11-CU:       ; %bb.0: ; %entry
5456; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5457; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5458; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
5459; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
5460; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
5461; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5462; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5463; GFX11-CU-NEXT:    ds_store_b32 v0, v1
5464; GFX11-CU-NEXT:    s_endpgm
5465    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5466entry:
5467  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5468  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire seq_cst
5469  %val0 = extractvalue { i32, i1 } %val, 0
5470  store i32 %val0, i32 addrspace(3)* %out, align 4
5471  ret void
5472}
5473
5474define amdgpu_kernel void @local_singlethread_release_seq_cst_ret_cmpxchg(
5475; GFX6-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5476; GFX6:       ; %bb.0: ; %entry
5477; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5478; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5479; GFX6-NEXT:    s_mov_b32 m0, -1
5480; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5481; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5482; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5483; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5484; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5485; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5486; GFX6-NEXT:    ds_write_b32 v0, v1
5487; GFX6-NEXT:    s_endpgm
5488;
5489; GFX7-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5490; GFX7:       ; %bb.0: ; %entry
5491; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5492; GFX7-NEXT:    s_mov_b32 m0, -1
5493; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5494; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5495; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5496; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5497; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5498; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5499; GFX7-NEXT:    ds_write_b32 v0, v1
5500; GFX7-NEXT:    s_endpgm
5501;
5502; GFX10-WGP-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5503; GFX10-WGP:       ; %bb.0: ; %entry
5504; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5505; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5506; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5507; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5508; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5509; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5510; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5511; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
5512; GFX10-WGP-NEXT:    s_endpgm
5513;
5514; GFX10-CU-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5515; GFX10-CU:       ; %bb.0: ; %entry
5516; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5517; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5518; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5519; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5520; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5521; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5522; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5523; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5524; GFX10-CU-NEXT:    s_endpgm
5525;
5526; SKIP-CACHE-INV-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5527; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5528; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5529; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5530; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5531; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5532; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5533; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5534; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5535; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5536; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
5537; SKIP-CACHE-INV-NEXT:    s_endpgm
5538;
5539; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5540; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5541; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5542; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5543; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5544; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5545; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5546; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5547; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5548; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5549; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5550;
5551; GFX90A-TGSPLIT-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5552; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5553; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5554; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5555; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5556; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5557; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5558; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5559; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5560; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5561; GFX90A-TGSPLIT-NEXT:    s_endpgm
5562;
5563; GFX940-NOTTGSPLIT-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5564; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5565; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5566; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5567; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5568; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5569; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5570; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5571; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5572; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5573; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5574;
5575; GFX940-TGSPLIT-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5576; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5577; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5578; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5579; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5580; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5581; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5582; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5583; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5584; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5585; GFX940-TGSPLIT-NEXT:    s_endpgm
5586;
5587; GFX11-WGP-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5588; GFX11-WGP:       ; %bb.0: ; %entry
5589; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5590; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5591; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
5592; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
5593; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
5594; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5595; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5596; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
5597; GFX11-WGP-NEXT:    s_endpgm
5598;
5599; GFX11-CU-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5600; GFX11-CU:       ; %bb.0: ; %entry
5601; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5602; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5603; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
5604; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
5605; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
5606; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5607; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5608; GFX11-CU-NEXT:    ds_store_b32 v0, v1
5609; GFX11-CU-NEXT:    s_endpgm
5610    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5611entry:
5612  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5613  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release seq_cst
5614  %val0 = extractvalue { i32, i1 } %val, 0
5615  store i32 %val0, i32 addrspace(3)* %out, align 4
5616  ret void
5617}
5618
5619define amdgpu_kernel void @local_singlethread_acq_rel_seq_cst_ret_cmpxchg(
5620; GFX6-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5621; GFX6:       ; %bb.0: ; %entry
5622; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5623; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5624; GFX6-NEXT:    s_mov_b32 m0, -1
5625; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5626; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5627; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5628; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5629; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5630; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5631; GFX6-NEXT:    ds_write_b32 v0, v1
5632; GFX6-NEXT:    s_endpgm
5633;
5634; GFX7-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5635; GFX7:       ; %bb.0: ; %entry
5636; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5637; GFX7-NEXT:    s_mov_b32 m0, -1
5638; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5639; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5640; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5641; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5642; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5643; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5644; GFX7-NEXT:    ds_write_b32 v0, v1
5645; GFX7-NEXT:    s_endpgm
5646;
5647; GFX10-WGP-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5648; GFX10-WGP:       ; %bb.0: ; %entry
5649; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5650; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5651; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5652; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5653; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5654; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5655; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5656; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
5657; GFX10-WGP-NEXT:    s_endpgm
5658;
5659; GFX10-CU-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5660; GFX10-CU:       ; %bb.0: ; %entry
5661; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5662; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5663; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5664; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5665; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5666; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5667; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5668; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5669; GFX10-CU-NEXT:    s_endpgm
5670;
5671; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5672; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5673; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5674; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5675; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5676; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5677; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5678; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5679; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5680; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5681; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
5682; SKIP-CACHE-INV-NEXT:    s_endpgm
5683;
5684; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5685; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5686; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5687; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5688; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5689; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5690; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5691; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5692; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5693; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5694; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5695;
5696; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5697; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5698; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5699; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5700; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5701; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5702; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5703; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5704; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5705; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5706; GFX90A-TGSPLIT-NEXT:    s_endpgm
5707;
5708; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5709; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5710; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5711; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5712; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5713; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5714; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5715; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5716; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5717; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5718; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5719;
5720; GFX940-TGSPLIT-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5721; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5722; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5723; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5724; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5725; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5726; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5727; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5728; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5729; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5730; GFX940-TGSPLIT-NEXT:    s_endpgm
5731;
5732; GFX11-WGP-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5733; GFX11-WGP:       ; %bb.0: ; %entry
5734; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5735; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5736; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
5737; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
5738; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
5739; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5740; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5741; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
5742; GFX11-WGP-NEXT:    s_endpgm
5743;
5744; GFX11-CU-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5745; GFX11-CU:       ; %bb.0: ; %entry
5746; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5747; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5748; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
5749; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
5750; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
5751; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5752; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5753; GFX11-CU-NEXT:    ds_store_b32 v0, v1
5754; GFX11-CU-NEXT:    s_endpgm
5755    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5756entry:
5757  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5758  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel seq_cst
5759  %val0 = extractvalue { i32, i1 } %val, 0
5760  store i32 %val0, i32 addrspace(3)* %out, align 4
5761  ret void
5762}
5763
5764define amdgpu_kernel void @local_singlethread_seq_cst_seq_cst_ret_cmpxchg(
5765; GFX6-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5766; GFX6:       ; %bb.0: ; %entry
5767; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5768; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5769; GFX6-NEXT:    s_mov_b32 m0, -1
5770; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5771; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5772; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5773; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5774; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5775; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5776; GFX6-NEXT:    ds_write_b32 v0, v1
5777; GFX6-NEXT:    s_endpgm
5778;
5779; GFX7-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5780; GFX7:       ; %bb.0: ; %entry
5781; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5782; GFX7-NEXT:    s_mov_b32 m0, -1
5783; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5784; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5785; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5786; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5787; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5788; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5789; GFX7-NEXT:    ds_write_b32 v0, v1
5790; GFX7-NEXT:    s_endpgm
5791;
5792; GFX10-WGP-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5793; GFX10-WGP:       ; %bb.0: ; %entry
5794; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5795; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5796; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5797; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5798; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5799; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5800; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5801; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
5802; GFX10-WGP-NEXT:    s_endpgm
5803;
5804; GFX10-CU-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5805; GFX10-CU:       ; %bb.0: ; %entry
5806; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5807; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5808; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5809; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5810; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5811; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5812; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5813; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5814; GFX10-CU-NEXT:    s_endpgm
5815;
5816; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5817; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5818; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5819; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5820; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5821; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5822; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5823; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5824; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5825; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5826; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
5827; SKIP-CACHE-INV-NEXT:    s_endpgm
5828;
5829; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5830; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5831; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5832; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5833; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5834; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5835; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5836; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5837; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5838; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5839; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5840;
5841; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5842; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5843; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5844; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5845; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5846; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5847; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5848; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5849; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5850; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5851; GFX90A-TGSPLIT-NEXT:    s_endpgm
5852;
5853; GFX940-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5854; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5855; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5856; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5857; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5858; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5859; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5860; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5861; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5862; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5863; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5864;
5865; GFX940-TGSPLIT-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5866; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5867; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5868; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5869; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5870; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5871; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5872; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5873; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5874; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5875; GFX940-TGSPLIT-NEXT:    s_endpgm
5876;
5877; GFX11-WGP-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5878; GFX11-WGP:       ; %bb.0: ; %entry
5879; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5880; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5881; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
5882; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
5883; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
5884; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5885; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5886; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
5887; GFX11-WGP-NEXT:    s_endpgm
5888;
5889; GFX11-CU-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5890; GFX11-CU:       ; %bb.0: ; %entry
5891; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5892; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5893; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
5894; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
5895; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
5896; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5897; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5898; GFX11-CU-NEXT:    ds_store_b32 v0, v1
5899; GFX11-CU-NEXT:    s_endpgm
5900    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5901entry:
5902  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5903  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
5904  %val0 = extractvalue { i32, i1 } %val, 0
5905  store i32 %val0, i32 addrspace(3)* %out, align 4
5906  ret void
5907}
5908
5909define amdgpu_kernel void @local_singlethread_one_as_unordered_load(
5910; GFX6-LABEL: local_singlethread_one_as_unordered_load:
5911; GFX6:       ; %bb.0: ; %entry
5912; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
5913; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
5914; GFX6-NEXT:    s_mov_b32 m0, -1
5915; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5916; GFX6-NEXT:    v_mov_b32_e32 v0, s0
5917; GFX6-NEXT:    ds_read_b32 v0, v0
5918; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5919; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5920; GFX6-NEXT:    ds_write_b32 v1, v0
5921; GFX6-NEXT:    s_endpgm
5922;
5923; GFX7-LABEL: local_singlethread_one_as_unordered_load:
5924; GFX7:       ; %bb.0: ; %entry
5925; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5926; GFX7-NEXT:    s_mov_b32 m0, -1
5927; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5928; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5929; GFX7-NEXT:    ds_read_b32 v0, v0
5930; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5931; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5932; GFX7-NEXT:    ds_write_b32 v1, v0
5933; GFX7-NEXT:    s_endpgm
5934;
5935; GFX10-WGP-LABEL: local_singlethread_one_as_unordered_load:
5936; GFX10-WGP:       ; %bb.0: ; %entry
5937; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5938; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5939; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5940; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5941; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
5942; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5943; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
5944; GFX10-WGP-NEXT:    s_endpgm
5945;
5946; GFX10-CU-LABEL: local_singlethread_one_as_unordered_load:
5947; GFX10-CU:       ; %bb.0: ; %entry
5948; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5949; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5950; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5951; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5952; GFX10-CU-NEXT:    ds_read_b32 v0, v0
5953; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5954; GFX10-CU-NEXT:    ds_write_b32 v1, v0
5955; GFX10-CU-NEXT:    s_endpgm
5956;
5957; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_unordered_load:
5958; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5959; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5960; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5961; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5962; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5963; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
5964; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
5965; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5966; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
5967; SKIP-CACHE-INV-NEXT:    s_endpgm
5968;
5969; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_unordered_load:
5970; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5971; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5972; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5973; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5974; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
5975; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5976; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5977; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
5978; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5979;
5980; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_unordered_load:
5981; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5982; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5983; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5984; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5985; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
5986; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5987; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5988; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
5989; GFX90A-TGSPLIT-NEXT:    s_endpgm
5990;
5991; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_unordered_load:
5992; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5993; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5994; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5995; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5996; GFX940-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
5997; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5998; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5999; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
6000; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6001;
6002; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_unordered_load:
6003; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6004; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6005; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6006; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6007; GFX940-TGSPLIT-NEXT:    ds_read_b32 v0, v0
6008; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6009; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6010; GFX940-TGSPLIT-NEXT:    ds_write_b32 v1, v0
6011; GFX940-TGSPLIT-NEXT:    s_endpgm
6012;
6013; GFX11-WGP-LABEL: local_singlethread_one_as_unordered_load:
6014; GFX11-WGP:       ; %bb.0: ; %entry
6015; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6016; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6017; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
6018; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
6019; GFX11-WGP-NEXT:    ds_load_b32 v0, v0
6020; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6021; GFX11-WGP-NEXT:    ds_store_b32 v1, v0
6022; GFX11-WGP-NEXT:    s_endpgm
6023;
6024; GFX11-CU-LABEL: local_singlethread_one_as_unordered_load:
6025; GFX11-CU:       ; %bb.0: ; %entry
6026; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6027; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6028; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
6029; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
6030; GFX11-CU-NEXT:    ds_load_b32 v0, v0
6031; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6032; GFX11-CU-NEXT:    ds_store_b32 v1, v0
6033; GFX11-CU-NEXT:    s_endpgm
6034    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
6035entry:
6036  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread-one-as") unordered, align 4
6037  store i32 %val, i32 addrspace(3)* %out
6038  ret void
6039}
6040
6041define amdgpu_kernel void @local_singlethread_one_as_monotonic_load(
6042; GFX6-LABEL: local_singlethread_one_as_monotonic_load:
6043; GFX6:       ; %bb.0: ; %entry
6044; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
6045; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
6046; GFX6-NEXT:    s_mov_b32 m0, -1
6047; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6048; GFX6-NEXT:    v_mov_b32_e32 v0, s0
6049; GFX6-NEXT:    ds_read_b32 v0, v0
6050; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6051; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6052; GFX6-NEXT:    ds_write_b32 v1, v0
6053; GFX6-NEXT:    s_endpgm
6054;
6055; GFX7-LABEL: local_singlethread_one_as_monotonic_load:
6056; GFX7:       ; %bb.0: ; %entry
6057; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6058; GFX7-NEXT:    s_mov_b32 m0, -1
6059; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6060; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6061; GFX7-NEXT:    ds_read_b32 v0, v0
6062; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6063; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6064; GFX7-NEXT:    ds_write_b32 v1, v0
6065; GFX7-NEXT:    s_endpgm
6066;
6067; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_load:
6068; GFX10-WGP:       ; %bb.0: ; %entry
6069; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6070; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6071; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6072; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6073; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
6074; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6075; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
6076; GFX10-WGP-NEXT:    s_endpgm
6077;
6078; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_load:
6079; GFX10-CU:       ; %bb.0: ; %entry
6080; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6081; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6082; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6083; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6084; GFX10-CU-NEXT:    ds_read_b32 v0, v0
6085; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6086; GFX10-CU-NEXT:    ds_write_b32 v1, v0
6087; GFX10-CU-NEXT:    s_endpgm
6088;
6089; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_load:
6090; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6091; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6092; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6093; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6094; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6095; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
6096; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6097; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6098; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
6099; SKIP-CACHE-INV-NEXT:    s_endpgm
6100;
6101; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_load:
6102; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6103; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6104; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6105; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6106; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
6107; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6108; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6109; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
6110; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6111;
6112; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_load:
6113; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6114; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6115; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6116; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6117; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
6118; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6119; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6120; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
6121; GFX90A-TGSPLIT-NEXT:    s_endpgm
6122;
6123; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_load:
6124; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6125; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6126; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6127; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6128; GFX940-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
6129; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6130; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6131; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
6132; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6133;
6134; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_load:
6135; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6136; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6137; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6138; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6139; GFX940-TGSPLIT-NEXT:    ds_read_b32 v0, v0
6140; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6141; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6142; GFX940-TGSPLIT-NEXT:    ds_write_b32 v1, v0
6143; GFX940-TGSPLIT-NEXT:    s_endpgm
6144;
6145; GFX11-WGP-LABEL: local_singlethread_one_as_monotonic_load:
6146; GFX11-WGP:       ; %bb.0: ; %entry
6147; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6148; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6149; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
6150; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
6151; GFX11-WGP-NEXT:    ds_load_b32 v0, v0
6152; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6153; GFX11-WGP-NEXT:    ds_store_b32 v1, v0
6154; GFX11-WGP-NEXT:    s_endpgm
6155;
6156; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_load:
6157; GFX11-CU:       ; %bb.0: ; %entry
6158; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6159; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6160; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
6161; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
6162; GFX11-CU-NEXT:    ds_load_b32 v0, v0
6163; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6164; GFX11-CU-NEXT:    ds_store_b32 v1, v0
6165; GFX11-CU-NEXT:    s_endpgm
6166    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
6167entry:
6168  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread-one-as") monotonic, align 4
6169  store i32 %val, i32 addrspace(3)* %out
6170  ret void
6171}
6172
6173define amdgpu_kernel void @local_singlethread_one_as_acquire_load(
6174; GFX6-LABEL: local_singlethread_one_as_acquire_load:
6175; GFX6:       ; %bb.0: ; %entry
6176; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
6177; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
6178; GFX6-NEXT:    s_mov_b32 m0, -1
6179; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6180; GFX6-NEXT:    v_mov_b32_e32 v0, s0
6181; GFX6-NEXT:    ds_read_b32 v0, v0
6182; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6183; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6184; GFX6-NEXT:    ds_write_b32 v1, v0
6185; GFX6-NEXT:    s_endpgm
6186;
6187; GFX7-LABEL: local_singlethread_one_as_acquire_load:
6188; GFX7:       ; %bb.0: ; %entry
6189; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6190; GFX7-NEXT:    s_mov_b32 m0, -1
6191; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6192; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6193; GFX7-NEXT:    ds_read_b32 v0, v0
6194; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6195; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6196; GFX7-NEXT:    ds_write_b32 v1, v0
6197; GFX7-NEXT:    s_endpgm
6198;
6199; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_load:
6200; GFX10-WGP:       ; %bb.0: ; %entry
6201; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6202; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6203; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6204; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6205; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
6206; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6207; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
6208; GFX10-WGP-NEXT:    s_endpgm
6209;
6210; GFX10-CU-LABEL: local_singlethread_one_as_acquire_load:
6211; GFX10-CU:       ; %bb.0: ; %entry
6212; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6213; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6214; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6215; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6216; GFX10-CU-NEXT:    ds_read_b32 v0, v0
6217; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6218; GFX10-CU-NEXT:    ds_write_b32 v1, v0
6219; GFX10-CU-NEXT:    s_endpgm
6220;
6221; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_load:
6222; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6223; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6224; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6225; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6226; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6227; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
6228; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6229; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6230; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
6231; SKIP-CACHE-INV-NEXT:    s_endpgm
6232;
6233; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_load:
6234; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6235; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6236; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6237; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6238; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
6239; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6240; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6241; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
6242; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6243;
6244; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_load:
6245; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6246; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6247; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6248; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6249; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
6250; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6251; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6252; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
6253; GFX90A-TGSPLIT-NEXT:    s_endpgm
6254;
6255; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_load:
6256; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6257; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6258; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6259; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6260; GFX940-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
6261; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6262; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6263; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
6264; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6265;
6266; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acquire_load:
6267; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6268; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6269; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6270; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6271; GFX940-TGSPLIT-NEXT:    ds_read_b32 v0, v0
6272; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6273; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6274; GFX940-TGSPLIT-NEXT:    ds_write_b32 v1, v0
6275; GFX940-TGSPLIT-NEXT:    s_endpgm
6276;
6277; GFX11-WGP-LABEL: local_singlethread_one_as_acquire_load:
6278; GFX11-WGP:       ; %bb.0: ; %entry
6279; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6280; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6281; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
6282; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
6283; GFX11-WGP-NEXT:    ds_load_b32 v0, v0
6284; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6285; GFX11-WGP-NEXT:    ds_store_b32 v1, v0
6286; GFX11-WGP-NEXT:    s_endpgm
6287;
6288; GFX11-CU-LABEL: local_singlethread_one_as_acquire_load:
6289; GFX11-CU:       ; %bb.0: ; %entry
6290; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6291; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6292; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
6293; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
6294; GFX11-CU-NEXT:    ds_load_b32 v0, v0
6295; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6296; GFX11-CU-NEXT:    ds_store_b32 v1, v0
6297; GFX11-CU-NEXT:    s_endpgm
6298    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
6299entry:
6300  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread-one-as") acquire, align 4
6301  store i32 %val, i32 addrspace(3)* %out
6302  ret void
6303}
6304
6305define amdgpu_kernel void @local_singlethread_one_as_seq_cst_load(
6306; GFX6-LABEL: local_singlethread_one_as_seq_cst_load:
6307; GFX6:       ; %bb.0: ; %entry
6308; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
6309; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
6310; GFX6-NEXT:    s_mov_b32 m0, -1
6311; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6312; GFX6-NEXT:    v_mov_b32_e32 v0, s0
6313; GFX6-NEXT:    ds_read_b32 v0, v0
6314; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6315; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6316; GFX6-NEXT:    ds_write_b32 v1, v0
6317; GFX6-NEXT:    s_endpgm
6318;
6319; GFX7-LABEL: local_singlethread_one_as_seq_cst_load:
6320; GFX7:       ; %bb.0: ; %entry
6321; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6322; GFX7-NEXT:    s_mov_b32 m0, -1
6323; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6324; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6325; GFX7-NEXT:    ds_read_b32 v0, v0
6326; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6327; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6328; GFX7-NEXT:    ds_write_b32 v1, v0
6329; GFX7-NEXT:    s_endpgm
6330;
6331; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_load:
6332; GFX10-WGP:       ; %bb.0: ; %entry
6333; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6334; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6335; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6336; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6337; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
6338; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6339; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
6340; GFX10-WGP-NEXT:    s_endpgm
6341;
6342; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_load:
6343; GFX10-CU:       ; %bb.0: ; %entry
6344; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6345; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6346; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6347; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6348; GFX10-CU-NEXT:    ds_read_b32 v0, v0
6349; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6350; GFX10-CU-NEXT:    ds_write_b32 v1, v0
6351; GFX10-CU-NEXT:    s_endpgm
6352;
6353; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_load:
6354; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6355; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6356; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6357; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6358; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6359; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
6360; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6361; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6362; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
6363; SKIP-CACHE-INV-NEXT:    s_endpgm
6364;
6365; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_load:
6366; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6367; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6368; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6369; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6370; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
6371; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6372; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6373; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
6374; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6375;
6376; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_load:
6377; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6378; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6379; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6380; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6381; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
6382; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6383; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6384; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
6385; GFX90A-TGSPLIT-NEXT:    s_endpgm
6386;
6387; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_load:
6388; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6389; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6390; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6391; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6392; GFX940-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
6393; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6394; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6395; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
6396; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6397;
6398; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_load:
6399; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6400; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6401; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6402; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6403; GFX940-TGSPLIT-NEXT:    ds_read_b32 v0, v0
6404; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6405; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6406; GFX940-TGSPLIT-NEXT:    ds_write_b32 v1, v0
6407; GFX940-TGSPLIT-NEXT:    s_endpgm
6408;
6409; GFX11-WGP-LABEL: local_singlethread_one_as_seq_cst_load:
6410; GFX11-WGP:       ; %bb.0: ; %entry
6411; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6412; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6413; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
6414; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
6415; GFX11-WGP-NEXT:    ds_load_b32 v0, v0
6416; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6417; GFX11-WGP-NEXT:    ds_store_b32 v1, v0
6418; GFX11-WGP-NEXT:    s_endpgm
6419;
6420; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_load:
6421; GFX11-CU:       ; %bb.0: ; %entry
6422; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6423; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6424; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
6425; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
6426; GFX11-CU-NEXT:    ds_load_b32 v0, v0
6427; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6428; GFX11-CU-NEXT:    ds_store_b32 v1, v0
6429; GFX11-CU-NEXT:    s_endpgm
6430    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
6431entry:
6432  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread-one-as") seq_cst, align 4
6433  store i32 %val, i32 addrspace(3)* %out
6434  ret void
6435}
6436
6437define amdgpu_kernel void @local_singlethread_one_as_unordered_store(
6438; GFX6-LABEL: local_singlethread_one_as_unordered_store:
6439; GFX6:       ; %bb.0: ; %entry
6440; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
6441; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
6442; GFX6-NEXT:    s_mov_b32 m0, -1
6443; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6444; GFX6-NEXT:    v_mov_b32_e32 v0, s0
6445; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6446; GFX6-NEXT:    ds_write_b32 v0, v1
6447; GFX6-NEXT:    s_endpgm
6448;
6449; GFX7-LABEL: local_singlethread_one_as_unordered_store:
6450; GFX7:       ; %bb.0: ; %entry
6451; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6452; GFX7-NEXT:    s_mov_b32 m0, -1
6453; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6454; GFX7-NEXT:    v_mov_b32_e32 v0, s1
6455; GFX7-NEXT:    v_mov_b32_e32 v1, s0
6456; GFX7-NEXT:    ds_write_b32 v0, v1
6457; GFX7-NEXT:    s_endpgm
6458;
6459; GFX10-WGP-LABEL: local_singlethread_one_as_unordered_store:
6460; GFX10-WGP:       ; %bb.0: ; %entry
6461; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6462; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6463; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
6464; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
6465; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
6466; GFX10-WGP-NEXT:    s_endpgm
6467;
6468; GFX10-CU-LABEL: local_singlethread_one_as_unordered_store:
6469; GFX10-CU:       ; %bb.0: ; %entry
6470; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6471; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6472; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
6473; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
6474; GFX10-CU-NEXT:    ds_write_b32 v0, v1
6475; GFX10-CU-NEXT:    s_endpgm
6476;
6477; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_unordered_store:
6478; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6479; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6480; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6481; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6482; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
6483; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
6484; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
6485; SKIP-CACHE-INV-NEXT:    s_endpgm
6486;
6487; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_unordered_store:
6488; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6489; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6490; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6491; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6492; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6493; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6494; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6495;
6496; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_unordered_store:
6497; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6498; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6499; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6500; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6501; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6502; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6503; GFX90A-TGSPLIT-NEXT:    s_endpgm
6504;
6505; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_unordered_store:
6506; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6507; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6508; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6509; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6510; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6511; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6512; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6513;
6514; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_unordered_store:
6515; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6516; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6517; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6518; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6519; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6520; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6521; GFX940-TGSPLIT-NEXT:    s_endpgm
6522;
6523; GFX11-WGP-LABEL: local_singlethread_one_as_unordered_store:
6524; GFX11-WGP:       ; %bb.0: ; %entry
6525; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6526; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6527; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s1
6528; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s0
6529; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
6530; GFX11-WGP-NEXT:    s_endpgm
6531;
6532; GFX11-CU-LABEL: local_singlethread_one_as_unordered_store:
6533; GFX11-CU:       ; %bb.0: ; %entry
6534; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6535; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6536; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s1
6537; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s0
6538; GFX11-CU-NEXT:    ds_store_b32 v0, v1
6539; GFX11-CU-NEXT:    s_endpgm
6540    i32 %in, i32 addrspace(3)* %out) {
6541entry:
6542  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread-one-as") unordered, align 4
6543  ret void
6544}
6545
6546define amdgpu_kernel void @local_singlethread_one_as_monotonic_store(
6547; GFX6-LABEL: local_singlethread_one_as_monotonic_store:
6548; GFX6:       ; %bb.0: ; %entry
6549; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
6550; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
6551; GFX6-NEXT:    s_mov_b32 m0, -1
6552; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6553; GFX6-NEXT:    v_mov_b32_e32 v0, s0
6554; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6555; GFX6-NEXT:    ds_write_b32 v0, v1
6556; GFX6-NEXT:    s_endpgm
6557;
6558; GFX7-LABEL: local_singlethread_one_as_monotonic_store:
6559; GFX7:       ; %bb.0: ; %entry
6560; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6561; GFX7-NEXT:    s_mov_b32 m0, -1
6562; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6563; GFX7-NEXT:    v_mov_b32_e32 v0, s1
6564; GFX7-NEXT:    v_mov_b32_e32 v1, s0
6565; GFX7-NEXT:    ds_write_b32 v0, v1
6566; GFX7-NEXT:    s_endpgm
6567;
6568; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_store:
6569; GFX10-WGP:       ; %bb.0: ; %entry
6570; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6571; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6572; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
6573; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
6574; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
6575; GFX10-WGP-NEXT:    s_endpgm
6576;
6577; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_store:
6578; GFX10-CU:       ; %bb.0: ; %entry
6579; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6580; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6581; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
6582; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
6583; GFX10-CU-NEXT:    ds_write_b32 v0, v1
6584; GFX10-CU-NEXT:    s_endpgm
6585;
6586; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_store:
6587; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6588; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6589; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6590; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6591; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
6592; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
6593; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
6594; SKIP-CACHE-INV-NEXT:    s_endpgm
6595;
6596; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_store:
6597; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6598; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6599; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6600; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6601; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6602; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6603; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6604;
6605; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_store:
6606; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6607; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6608; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6609; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6610; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6611; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6612; GFX90A-TGSPLIT-NEXT:    s_endpgm
6613;
6614; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_store:
6615; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6616; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6617; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6618; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6619; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6620; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6621; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6622;
6623; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_store:
6624; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6625; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6626; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6627; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6628; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6629; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6630; GFX940-TGSPLIT-NEXT:    s_endpgm
6631;
6632; GFX11-WGP-LABEL: local_singlethread_one_as_monotonic_store:
6633; GFX11-WGP:       ; %bb.0: ; %entry
6634; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6635; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6636; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s1
6637; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s0
6638; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
6639; GFX11-WGP-NEXT:    s_endpgm
6640;
6641; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_store:
6642; GFX11-CU:       ; %bb.0: ; %entry
6643; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6644; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6645; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s1
6646; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s0
6647; GFX11-CU-NEXT:    ds_store_b32 v0, v1
6648; GFX11-CU-NEXT:    s_endpgm
6649    i32 %in, i32 addrspace(3)* %out) {
6650entry:
6651  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread-one-as") monotonic, align 4
6652  ret void
6653}
6654
6655define amdgpu_kernel void @local_singlethread_one_as_release_store(
6656; GFX6-LABEL: local_singlethread_one_as_release_store:
6657; GFX6:       ; %bb.0: ; %entry
6658; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
6659; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
6660; GFX6-NEXT:    s_mov_b32 m0, -1
6661; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6662; GFX6-NEXT:    v_mov_b32_e32 v0, s0
6663; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6664; GFX6-NEXT:    ds_write_b32 v0, v1
6665; GFX6-NEXT:    s_endpgm
6666;
6667; GFX7-LABEL: local_singlethread_one_as_release_store:
6668; GFX7:       ; %bb.0: ; %entry
6669; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6670; GFX7-NEXT:    s_mov_b32 m0, -1
6671; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6672; GFX7-NEXT:    v_mov_b32_e32 v0, s1
6673; GFX7-NEXT:    v_mov_b32_e32 v1, s0
6674; GFX7-NEXT:    ds_write_b32 v0, v1
6675; GFX7-NEXT:    s_endpgm
6676;
6677; GFX10-WGP-LABEL: local_singlethread_one_as_release_store:
6678; GFX10-WGP:       ; %bb.0: ; %entry
6679; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6680; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6681; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
6682; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
6683; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
6684; GFX10-WGP-NEXT:    s_endpgm
6685;
6686; GFX10-CU-LABEL: local_singlethread_one_as_release_store:
6687; GFX10-CU:       ; %bb.0: ; %entry
6688; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6689; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6690; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
6691; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
6692; GFX10-CU-NEXT:    ds_write_b32 v0, v1
6693; GFX10-CU-NEXT:    s_endpgm
6694;
6695; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_store:
6696; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6697; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6698; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6699; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6700; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
6701; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
6702; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
6703; SKIP-CACHE-INV-NEXT:    s_endpgm
6704;
6705; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_store:
6706; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6707; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6708; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6709; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6710; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6711; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6712; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6713;
6714; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_store:
6715; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6716; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6717; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6718; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6719; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6720; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6721; GFX90A-TGSPLIT-NEXT:    s_endpgm
6722;
6723; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_store:
6724; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6725; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6726; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6727; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6728; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6729; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6730; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6731;
6732; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_release_store:
6733; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6734; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6735; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6736; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6737; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6738; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6739; GFX940-TGSPLIT-NEXT:    s_endpgm
6740;
6741; GFX11-WGP-LABEL: local_singlethread_one_as_release_store:
6742; GFX11-WGP:       ; %bb.0: ; %entry
6743; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6744; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6745; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s1
6746; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s0
6747; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
6748; GFX11-WGP-NEXT:    s_endpgm
6749;
6750; GFX11-CU-LABEL: local_singlethread_one_as_release_store:
6751; GFX11-CU:       ; %bb.0: ; %entry
6752; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6753; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6754; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s1
6755; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s0
6756; GFX11-CU-NEXT:    ds_store_b32 v0, v1
6757; GFX11-CU-NEXT:    s_endpgm
6758    i32 %in, i32 addrspace(3)* %out) {
6759entry:
6760  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread-one-as") release, align 4
6761  ret void
6762}
6763
6764define amdgpu_kernel void @local_singlethread_one_as_seq_cst_store(
6765; GFX6-LABEL: local_singlethread_one_as_seq_cst_store:
6766; GFX6:       ; %bb.0: ; %entry
6767; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
6768; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
6769; GFX6-NEXT:    s_mov_b32 m0, -1
6770; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6771; GFX6-NEXT:    v_mov_b32_e32 v0, s0
6772; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6773; GFX6-NEXT:    ds_write_b32 v0, v1
6774; GFX6-NEXT:    s_endpgm
6775;
6776; GFX7-LABEL: local_singlethread_one_as_seq_cst_store:
6777; GFX7:       ; %bb.0: ; %entry
6778; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6779; GFX7-NEXT:    s_mov_b32 m0, -1
6780; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6781; GFX7-NEXT:    v_mov_b32_e32 v0, s1
6782; GFX7-NEXT:    v_mov_b32_e32 v1, s0
6783; GFX7-NEXT:    ds_write_b32 v0, v1
6784; GFX7-NEXT:    s_endpgm
6785;
6786; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_store:
6787; GFX10-WGP:       ; %bb.0: ; %entry
6788; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6789; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6790; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
6791; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
6792; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
6793; GFX10-WGP-NEXT:    s_endpgm
6794;
6795; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_store:
6796; GFX10-CU:       ; %bb.0: ; %entry
6797; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6798; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6799; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
6800; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
6801; GFX10-CU-NEXT:    ds_write_b32 v0, v1
6802; GFX10-CU-NEXT:    s_endpgm
6803;
6804; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_store:
6805; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6806; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6807; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6808; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6809; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
6810; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
6811; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
6812; SKIP-CACHE-INV-NEXT:    s_endpgm
6813;
6814; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_store:
6815; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6816; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6817; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6818; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6819; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6820; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6821; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6822;
6823; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_store:
6824; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6825; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6826; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6827; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6828; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6829; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6830; GFX90A-TGSPLIT-NEXT:    s_endpgm
6831;
6832; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_store:
6833; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6834; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6835; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6836; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6837; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6838; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6839; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6840;
6841; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_store:
6842; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6843; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6844; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6845; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6846; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6847; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6848; GFX940-TGSPLIT-NEXT:    s_endpgm
6849;
6850; GFX11-WGP-LABEL: local_singlethread_one_as_seq_cst_store:
6851; GFX11-WGP:       ; %bb.0: ; %entry
6852; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6853; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6854; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s1
6855; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s0
6856; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
6857; GFX11-WGP-NEXT:    s_endpgm
6858;
6859; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_store:
6860; GFX11-CU:       ; %bb.0: ; %entry
6861; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6862; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6863; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s1
6864; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s0
6865; GFX11-CU-NEXT:    ds_store_b32 v0, v1
6866; GFX11-CU-NEXT:    s_endpgm
6867    i32 %in, i32 addrspace(3)* %out) {
6868entry:
6869  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread-one-as") seq_cst, align 4
6870  ret void
6871}
6872
6873define amdgpu_kernel void @local_singlethread_one_as_monotonic_atomicrmw(
6874; GFX6-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6875; GFX6:       ; %bb.0: ; %entry
6876; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
6877; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
6878; GFX6-NEXT:    s_mov_b32 m0, -1
6879; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6880; GFX6-NEXT:    v_mov_b32_e32 v0, s0
6881; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6882; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6883; GFX6-NEXT:    s_endpgm
6884;
6885; GFX7-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6886; GFX7:       ; %bb.0: ; %entry
6887; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6888; GFX7-NEXT:    s_mov_b32 m0, -1
6889; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6890; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6891; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6892; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6893; GFX7-NEXT:    s_endpgm
6894;
6895; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6896; GFX10-WGP:       ; %bb.0: ; %entry
6897; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6898; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6899; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6900; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6901; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6902; GFX10-WGP-NEXT:    s_endpgm
6903;
6904; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6905; GFX10-CU:       ; %bb.0: ; %entry
6906; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6907; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6908; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6909; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6910; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6911; GFX10-CU-NEXT:    s_endpgm
6912;
6913; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6914; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6915; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6916; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6917; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6918; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6919; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6920; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6921; SKIP-CACHE-INV-NEXT:    s_endpgm
6922;
6923; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6924; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6925; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6926; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6927; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6928; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6929; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6930; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6931;
6932; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6933; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6934; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6935; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6936; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6937; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6938; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6939; GFX90A-TGSPLIT-NEXT:    s_endpgm
6940;
6941; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6942; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6943; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6944; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6945; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6946; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6947; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6948; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6949;
6950; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6951; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6952; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6953; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6954; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6955; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6956; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6957; GFX940-TGSPLIT-NEXT:    s_endpgm
6958;
6959; GFX11-WGP-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6960; GFX11-WGP:       ; %bb.0: ; %entry
6961; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6962; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6963; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
6964; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
6965; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
6966; GFX11-WGP-NEXT:    s_endpgm
6967;
6968; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6969; GFX11-CU:       ; %bb.0: ; %entry
6970; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6971; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6972; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
6973; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
6974; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
6975; GFX11-CU-NEXT:    s_endpgm
6976    i32 addrspace(3)* %out, i32 %in) {
6977entry:
6978  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") monotonic
6979  ret void
6980}
6981
6982define amdgpu_kernel void @local_singlethread_one_as_acquire_atomicrmw(
6983; GFX6-LABEL: local_singlethread_one_as_acquire_atomicrmw:
6984; GFX6:       ; %bb.0: ; %entry
6985; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
6986; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
6987; GFX6-NEXT:    s_mov_b32 m0, -1
6988; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6989; GFX6-NEXT:    v_mov_b32_e32 v0, s0
6990; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6991; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6992; GFX6-NEXT:    s_endpgm
6993;
6994; GFX7-LABEL: local_singlethread_one_as_acquire_atomicrmw:
6995; GFX7:       ; %bb.0: ; %entry
6996; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6997; GFX7-NEXT:    s_mov_b32 m0, -1
6998; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6999; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7000; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7001; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7002; GFX7-NEXT:    s_endpgm
7003;
7004; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_atomicrmw:
7005; GFX10-WGP:       ; %bb.0: ; %entry
7006; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7007; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7008; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7009; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7010; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7011; GFX10-WGP-NEXT:    s_endpgm
7012;
7013; GFX10-CU-LABEL: local_singlethread_one_as_acquire_atomicrmw:
7014; GFX10-CU:       ; %bb.0: ; %entry
7015; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7016; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7017; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7018; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7019; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7020; GFX10-CU-NEXT:    s_endpgm
7021;
7022; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_atomicrmw:
7023; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7024; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7025; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7026; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7027; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7028; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7029; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7030; SKIP-CACHE-INV-NEXT:    s_endpgm
7031;
7032; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_atomicrmw:
7033; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7034; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7035; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7036; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7037; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7038; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7039; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7040;
7041; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_atomicrmw:
7042; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7043; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7044; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7045; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7046; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7047; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7048; GFX90A-TGSPLIT-NEXT:    s_endpgm
7049;
7050; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_atomicrmw:
7051; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7052; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7053; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7054; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7055; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7056; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7057; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7058;
7059; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acquire_atomicrmw:
7060; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7061; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7062; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7063; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7064; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7065; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7066; GFX940-TGSPLIT-NEXT:    s_endpgm
7067;
7068; GFX11-WGP-LABEL: local_singlethread_one_as_acquire_atomicrmw:
7069; GFX11-WGP:       ; %bb.0: ; %entry
7070; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7071; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7072; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
7073; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
7074; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
7075; GFX11-WGP-NEXT:    s_endpgm
7076;
7077; GFX11-CU-LABEL: local_singlethread_one_as_acquire_atomicrmw:
7078; GFX11-CU:       ; %bb.0: ; %entry
7079; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7080; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7081; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
7082; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
7083; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
7084; GFX11-CU-NEXT:    s_endpgm
7085    i32 addrspace(3)* %out, i32 %in) {
7086entry:
7087  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") acquire
7088  ret void
7089}
7090
7091define amdgpu_kernel void @local_singlethread_one_as_release_atomicrmw(
7092; GFX6-LABEL: local_singlethread_one_as_release_atomicrmw:
7093; GFX6:       ; %bb.0: ; %entry
7094; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
7095; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
7096; GFX6-NEXT:    s_mov_b32 m0, -1
7097; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7098; GFX6-NEXT:    v_mov_b32_e32 v0, s0
7099; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7100; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7101; GFX6-NEXT:    s_endpgm
7102;
7103; GFX7-LABEL: local_singlethread_one_as_release_atomicrmw:
7104; GFX7:       ; %bb.0: ; %entry
7105; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7106; GFX7-NEXT:    s_mov_b32 m0, -1
7107; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7108; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7109; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7110; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7111; GFX7-NEXT:    s_endpgm
7112;
7113; GFX10-WGP-LABEL: local_singlethread_one_as_release_atomicrmw:
7114; GFX10-WGP:       ; %bb.0: ; %entry
7115; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7116; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7117; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7118; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7119; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7120; GFX10-WGP-NEXT:    s_endpgm
7121;
7122; GFX10-CU-LABEL: local_singlethread_one_as_release_atomicrmw:
7123; GFX10-CU:       ; %bb.0: ; %entry
7124; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7125; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7126; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7127; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7128; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7129; GFX10-CU-NEXT:    s_endpgm
7130;
7131; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_atomicrmw:
7132; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7133; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7134; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7135; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7136; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7137; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7138; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7139; SKIP-CACHE-INV-NEXT:    s_endpgm
7140;
7141; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_atomicrmw:
7142; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7143; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7144; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7145; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7146; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7147; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7148; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7149;
7150; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_atomicrmw:
7151; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7152; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7153; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7154; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7155; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7156; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7157; GFX90A-TGSPLIT-NEXT:    s_endpgm
7158;
7159; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_atomicrmw:
7160; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7161; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7162; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7163; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7164; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7165; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7166; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7167;
7168; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_release_atomicrmw:
7169; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7170; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7171; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7172; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7173; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7174; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7175; GFX940-TGSPLIT-NEXT:    s_endpgm
7176;
7177; GFX11-WGP-LABEL: local_singlethread_one_as_release_atomicrmw:
7178; GFX11-WGP:       ; %bb.0: ; %entry
7179; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7180; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7181; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
7182; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
7183; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
7184; GFX11-WGP-NEXT:    s_endpgm
7185;
7186; GFX11-CU-LABEL: local_singlethread_one_as_release_atomicrmw:
7187; GFX11-CU:       ; %bb.0: ; %entry
7188; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7189; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7190; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
7191; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
7192; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
7193; GFX11-CU-NEXT:    s_endpgm
7194    i32 addrspace(3)* %out, i32 %in) {
7195entry:
7196  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") release
7197  ret void
7198}
7199
7200define amdgpu_kernel void @local_singlethread_one_as_acq_rel_atomicrmw(
7201; GFX6-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7202; GFX6:       ; %bb.0: ; %entry
7203; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
7204; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
7205; GFX6-NEXT:    s_mov_b32 m0, -1
7206; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7207; GFX6-NEXT:    v_mov_b32_e32 v0, s0
7208; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7209; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7210; GFX6-NEXT:    s_endpgm
7211;
7212; GFX7-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7213; GFX7:       ; %bb.0: ; %entry
7214; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7215; GFX7-NEXT:    s_mov_b32 m0, -1
7216; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7217; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7218; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7219; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7220; GFX7-NEXT:    s_endpgm
7221;
7222; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7223; GFX10-WGP:       ; %bb.0: ; %entry
7224; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7225; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7226; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7227; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7228; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7229; GFX10-WGP-NEXT:    s_endpgm
7230;
7231; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7232; GFX10-CU:       ; %bb.0: ; %entry
7233; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7234; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7235; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7236; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7237; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7238; GFX10-CU-NEXT:    s_endpgm
7239;
7240; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7241; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7242; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7243; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7244; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7245; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7246; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7247; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7248; SKIP-CACHE-INV-NEXT:    s_endpgm
7249;
7250; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7251; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7252; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7253; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7254; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7255; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7256; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7257; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7258;
7259; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7260; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7261; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7262; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7263; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7264; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7265; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7266; GFX90A-TGSPLIT-NEXT:    s_endpgm
7267;
7268; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7269; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7270; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7271; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7272; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7273; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7274; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7275; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7276;
7277; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7278; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7279; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7280; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7281; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7282; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7283; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7284; GFX940-TGSPLIT-NEXT:    s_endpgm
7285;
7286; GFX11-WGP-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7287; GFX11-WGP:       ; %bb.0: ; %entry
7288; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7289; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7290; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
7291; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
7292; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
7293; GFX11-WGP-NEXT:    s_endpgm
7294;
7295; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7296; GFX11-CU:       ; %bb.0: ; %entry
7297; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7298; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7299; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
7300; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
7301; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
7302; GFX11-CU-NEXT:    s_endpgm
7303    i32 addrspace(3)* %out, i32 %in) {
7304entry:
7305  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") acq_rel
7306  ret void
7307}
7308
7309define amdgpu_kernel void @local_singlethread_one_as_seq_cst_atomicrmw(
7310; GFX6-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7311; GFX6:       ; %bb.0: ; %entry
7312; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
7313; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
7314; GFX6-NEXT:    s_mov_b32 m0, -1
7315; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7316; GFX6-NEXT:    v_mov_b32_e32 v0, s0
7317; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7318; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7319; GFX6-NEXT:    s_endpgm
7320;
7321; GFX7-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7322; GFX7:       ; %bb.0: ; %entry
7323; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7324; GFX7-NEXT:    s_mov_b32 m0, -1
7325; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7326; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7327; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7328; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7329; GFX7-NEXT:    s_endpgm
7330;
7331; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7332; GFX10-WGP:       ; %bb.0: ; %entry
7333; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7334; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7335; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7336; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7337; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7338; GFX10-WGP-NEXT:    s_endpgm
7339;
7340; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7341; GFX10-CU:       ; %bb.0: ; %entry
7342; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7343; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7344; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7345; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7346; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7347; GFX10-CU-NEXT:    s_endpgm
7348;
7349; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7350; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7351; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7352; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7353; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7354; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7355; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7356; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7357; SKIP-CACHE-INV-NEXT:    s_endpgm
7358;
7359; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7360; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7361; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7362; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7363; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7364; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7365; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7366; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7367;
7368; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7369; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7370; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7371; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7372; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7373; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7374; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7375; GFX90A-TGSPLIT-NEXT:    s_endpgm
7376;
7377; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7378; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7379; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7380; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7381; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7382; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7383; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7384; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7385;
7386; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7387; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7388; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7389; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7390; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7391; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7392; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7393; GFX940-TGSPLIT-NEXT:    s_endpgm
7394;
7395; GFX11-WGP-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7396; GFX11-WGP:       ; %bb.0: ; %entry
7397; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7398; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7399; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
7400; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
7401; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
7402; GFX11-WGP-NEXT:    s_endpgm
7403;
7404; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7405; GFX11-CU:       ; %bb.0: ; %entry
7406; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7407; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7408; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
7409; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
7410; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
7411; GFX11-CU-NEXT:    s_endpgm
7412    i32 addrspace(3)* %out, i32 %in) {
7413entry:
7414  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") seq_cst
7415  ret void
7416}
7417
7418define amdgpu_kernel void @local_singlethread_one_as_acquire_ret_atomicrmw(
7419; GFX6-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7420; GFX6:       ; %bb.0: ; %entry
7421; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
7422; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
7423; GFX6-NEXT:    s_mov_b32 m0, -1
7424; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7425; GFX6-NEXT:    v_mov_b32_e32 v0, s0
7426; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7427; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7428; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7429; GFX6-NEXT:    ds_write_b32 v0, v1
7430; GFX6-NEXT:    s_endpgm
7431;
7432; GFX7-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7433; GFX7:       ; %bb.0: ; %entry
7434; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7435; GFX7-NEXT:    s_mov_b32 m0, -1
7436; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7437; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7438; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7439; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7440; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7441; GFX7-NEXT:    ds_write_b32 v0, v1
7442; GFX7-NEXT:    s_endpgm
7443;
7444; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7445; GFX10-WGP:       ; %bb.0: ; %entry
7446; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7447; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7448; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7449; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7450; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7451; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7452; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
7453; GFX10-WGP-NEXT:    s_endpgm
7454;
7455; GFX10-CU-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7456; GFX10-CU:       ; %bb.0: ; %entry
7457; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7458; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7459; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7460; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7461; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7462; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7463; GFX10-CU-NEXT:    ds_write_b32 v0, v1
7464; GFX10-CU-NEXT:    s_endpgm
7465;
7466; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7467; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7468; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7469; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7470; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7471; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7472; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7473; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7474; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7475; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
7476; SKIP-CACHE-INV-NEXT:    s_endpgm
7477;
7478; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7479; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7480; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7481; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7482; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7483; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7484; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7485; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7486; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7487; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7488;
7489; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7490; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7491; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7492; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7493; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7494; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7495; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7496; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7497; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7498; GFX90A-TGSPLIT-NEXT:    s_endpgm
7499;
7500; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7501; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7502; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7503; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7504; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7505; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7506; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7507; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7508; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7509; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7510;
7511; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7512; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7513; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7514; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7515; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7516; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7517; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7518; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7519; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7520; GFX940-TGSPLIT-NEXT:    s_endpgm
7521;
7522; GFX11-WGP-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7523; GFX11-WGP:       ; %bb.0: ; %entry
7524; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7525; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7526; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
7527; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
7528; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
7529; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7530; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
7531; GFX11-WGP-NEXT:    s_endpgm
7532;
7533; GFX11-CU-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7534; GFX11-CU:       ; %bb.0: ; %entry
7535; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7536; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7537; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
7538; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
7539; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
7540; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7541; GFX11-CU-NEXT:    ds_store_b32 v0, v1
7542; GFX11-CU-NEXT:    s_endpgm
7543    i32 addrspace(3)* %out, i32 %in) {
7544entry:
7545  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") acquire
7546  store i32 %val, i32 addrspace(3)* %out, align 4
7547  ret void
7548}
7549
7550define amdgpu_kernel void @local_singlethread_one_as_acq_rel_ret_atomicrmw(
7551; GFX6-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7552; GFX6:       ; %bb.0: ; %entry
7553; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
7554; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
7555; GFX6-NEXT:    s_mov_b32 m0, -1
7556; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7557; GFX6-NEXT:    v_mov_b32_e32 v0, s0
7558; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7559; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7560; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7561; GFX6-NEXT:    ds_write_b32 v0, v1
7562; GFX6-NEXT:    s_endpgm
7563;
7564; GFX7-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7565; GFX7:       ; %bb.0: ; %entry
7566; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7567; GFX7-NEXT:    s_mov_b32 m0, -1
7568; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7569; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7570; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7571; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7572; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7573; GFX7-NEXT:    ds_write_b32 v0, v1
7574; GFX7-NEXT:    s_endpgm
7575;
7576; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7577; GFX10-WGP:       ; %bb.0: ; %entry
7578; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7579; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7580; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7581; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7582; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7583; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7584; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
7585; GFX10-WGP-NEXT:    s_endpgm
7586;
7587; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7588; GFX10-CU:       ; %bb.0: ; %entry
7589; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7590; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7591; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7592; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7593; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7594; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7595; GFX10-CU-NEXT:    ds_write_b32 v0, v1
7596; GFX10-CU-NEXT:    s_endpgm
7597;
7598; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7599; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7600; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7601; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7602; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7603; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7604; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7605; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7606; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7607; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
7608; SKIP-CACHE-INV-NEXT:    s_endpgm
7609;
7610; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7611; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7612; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7613; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7614; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7615; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7616; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7617; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7618; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7619; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7620;
7621; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7622; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7623; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7624; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7625; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7626; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7627; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7628; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7629; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7630; GFX90A-TGSPLIT-NEXT:    s_endpgm
7631;
7632; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7633; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7634; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7635; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7636; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7637; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7638; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7639; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7640; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7641; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7642;
7643; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7644; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7645; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7646; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7647; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7648; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7649; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7650; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7651; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7652; GFX940-TGSPLIT-NEXT:    s_endpgm
7653;
7654; GFX11-WGP-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7655; GFX11-WGP:       ; %bb.0: ; %entry
7656; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7657; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7658; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
7659; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
7660; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
7661; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7662; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
7663; GFX11-WGP-NEXT:    s_endpgm
7664;
7665; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7666; GFX11-CU:       ; %bb.0: ; %entry
7667; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7668; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7669; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
7670; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
7671; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
7672; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7673; GFX11-CU-NEXT:    ds_store_b32 v0, v1
7674; GFX11-CU-NEXT:    s_endpgm
7675    i32 addrspace(3)* %out, i32 %in) {
7676entry:
7677  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") acq_rel
7678  store i32 %val, i32 addrspace(3)* %out, align 4
7679  ret void
7680}
7681
7682define amdgpu_kernel void @local_singlethread_one_as_seq_cst_ret_atomicrmw(
7683; GFX6-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7684; GFX6:       ; %bb.0: ; %entry
7685; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
7686; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
7687; GFX6-NEXT:    s_mov_b32 m0, -1
7688; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7689; GFX6-NEXT:    v_mov_b32_e32 v0, s0
7690; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7691; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7692; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7693; GFX6-NEXT:    ds_write_b32 v0, v1
7694; GFX6-NEXT:    s_endpgm
7695;
7696; GFX7-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7697; GFX7:       ; %bb.0: ; %entry
7698; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7699; GFX7-NEXT:    s_mov_b32 m0, -1
7700; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7701; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7702; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7703; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7704; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7705; GFX7-NEXT:    ds_write_b32 v0, v1
7706; GFX7-NEXT:    s_endpgm
7707;
7708; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7709; GFX10-WGP:       ; %bb.0: ; %entry
7710; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7711; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7712; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7713; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7714; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7715; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7716; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
7717; GFX10-WGP-NEXT:    s_endpgm
7718;
7719; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7720; GFX10-CU:       ; %bb.0: ; %entry
7721; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7722; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7723; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7724; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7725; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7726; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7727; GFX10-CU-NEXT:    ds_write_b32 v0, v1
7728; GFX10-CU-NEXT:    s_endpgm
7729;
7730; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7731; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7732; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7733; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7734; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7735; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7736; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7737; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7738; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7739; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
7740; SKIP-CACHE-INV-NEXT:    s_endpgm
7741;
7742; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7743; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7744; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7745; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7746; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7747; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7748; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7749; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7750; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7751; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7752;
7753; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7754; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7755; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7756; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7757; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7758; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7759; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7760; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7761; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7762; GFX90A-TGSPLIT-NEXT:    s_endpgm
7763;
7764; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7765; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7766; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7767; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7768; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7769; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7770; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7771; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7772; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7773; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7774;
7775; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7776; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7777; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7778; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7779; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7780; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7781; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7782; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7783; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7784; GFX940-TGSPLIT-NEXT:    s_endpgm
7785;
7786; GFX11-WGP-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7787; GFX11-WGP:       ; %bb.0: ; %entry
7788; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7789; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7790; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
7791; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
7792; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
7793; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7794; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
7795; GFX11-WGP-NEXT:    s_endpgm
7796;
7797; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7798; GFX11-CU:       ; %bb.0: ; %entry
7799; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7800; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7801; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
7802; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
7803; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
7804; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7805; GFX11-CU-NEXT:    ds_store_b32 v0, v1
7806; GFX11-CU-NEXT:    s_endpgm
7807    i32 addrspace(3)* %out, i32 %in) {
7808entry:
7809  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") seq_cst
7810  store i32 %val, i32 addrspace(3)* %out, align 4
7811  ret void
7812}
7813
7814define amdgpu_kernel void @local_singlethread_one_as_monotonic_monotonic_cmpxchg(
7815; GFX6-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7816; GFX6:       ; %bb.0: ; %entry
7817; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
7818; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
7819; GFX6-NEXT:    s_mov_b32 m0, -1
7820; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7821; GFX6-NEXT:    v_mov_b32_e32 v0, s2
7822; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7823; GFX6-NEXT:    v_mov_b32_e32 v2, s0
7824; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7825; GFX6-NEXT:    s_endpgm
7826;
7827; GFX7-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7828; GFX7:       ; %bb.0: ; %entry
7829; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7830; GFX7-NEXT:    s_mov_b32 m0, -1
7831; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7832; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7833; GFX7-NEXT:    v_mov_b32_e32 v1, s2
7834; GFX7-NEXT:    v_mov_b32_e32 v2, s1
7835; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7836; GFX7-NEXT:    s_endpgm
7837;
7838; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7839; GFX10-WGP:       ; %bb.0: ; %entry
7840; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7841; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7842; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7843; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7844; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
7845; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7846; GFX10-WGP-NEXT:    s_endpgm
7847;
7848; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7849; GFX10-CU:       ; %bb.0: ; %entry
7850; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7851; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7852; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7853; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7854; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
7855; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7856; GFX10-CU-NEXT:    s_endpgm
7857;
7858; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7859; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7860; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7861; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7862; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7863; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7864; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
7865; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
7866; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7867; SKIP-CACHE-INV-NEXT:    s_endpgm
7868;
7869; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7870; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7871; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7872; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7873; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7874; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7875; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7876; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7877; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7878;
7879; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7880; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7881; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7882; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7883; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7884; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7885; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7886; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7887; GFX90A-TGSPLIT-NEXT:    s_endpgm
7888;
7889; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7890; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7891; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7892; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7893; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7894; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7895; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7896; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7897; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7898;
7899; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7900; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7901; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7902; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7903; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7904; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7905; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7906; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7907; GFX940-TGSPLIT-NEXT:    s_endpgm
7908;
7909; GFX11-WGP-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7910; GFX11-WGP:       ; %bb.0: ; %entry
7911; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7912; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7913; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
7914; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
7915; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
7916; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
7917; GFX11-WGP-NEXT:    s_endpgm
7918;
7919; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7920; GFX11-CU:       ; %bb.0: ; %entry
7921; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7922; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7923; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
7924; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
7925; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
7926; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
7927; GFX11-CU-NEXT:    s_endpgm
7928    i32 addrspace(3)* %out, i32 %in, i32 %old) {
7929entry:
7930  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
7931  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
7932  ret void
7933}
7934
7935define amdgpu_kernel void @local_singlethread_one_as_acquire_monotonic_cmpxchg(
7936; GFX6-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
7937; GFX6:       ; %bb.0: ; %entry
7938; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
7939; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
7940; GFX6-NEXT:    s_mov_b32 m0, -1
7941; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7942; GFX6-NEXT:    v_mov_b32_e32 v0, s2
7943; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7944; GFX6-NEXT:    v_mov_b32_e32 v2, s0
7945; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7946; GFX6-NEXT:    s_endpgm
7947;
7948; GFX7-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
7949; GFX7:       ; %bb.0: ; %entry
7950; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7951; GFX7-NEXT:    s_mov_b32 m0, -1
7952; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7953; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7954; GFX7-NEXT:    v_mov_b32_e32 v1, s2
7955; GFX7-NEXT:    v_mov_b32_e32 v2, s1
7956; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7957; GFX7-NEXT:    s_endpgm
7958;
7959; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
7960; GFX10-WGP:       ; %bb.0: ; %entry
7961; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7962; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7963; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7964; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7965; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
7966; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7967; GFX10-WGP-NEXT:    s_endpgm
7968;
7969; GFX10-CU-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
7970; GFX10-CU:       ; %bb.0: ; %entry
7971; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7972; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7973; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7974; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7975; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
7976; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7977; GFX10-CU-NEXT:    s_endpgm
7978;
7979; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
7980; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7981; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7982; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7983; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7984; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7985; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
7986; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
7987; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7988; SKIP-CACHE-INV-NEXT:    s_endpgm
7989;
7990; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
7991; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7992; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7993; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7994; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7995; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7996; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7997; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7998; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7999;
8000; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
8001; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8002; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8003; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8004; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8005; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8006; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8007; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8008; GFX90A-TGSPLIT-NEXT:    s_endpgm
8009;
8010; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
8011; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8012; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8013; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8014; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8015; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8016; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8017; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8018; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8019;
8020; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
8021; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8022; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8023; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8024; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8025; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8026; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8027; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8028; GFX940-TGSPLIT-NEXT:    s_endpgm
8029;
8030; GFX11-WGP-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
8031; GFX11-WGP:       ; %bb.0: ; %entry
8032; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8033; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8034; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
8035; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
8036; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
8037; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8038; GFX11-WGP-NEXT:    s_endpgm
8039;
8040; GFX11-CU-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
8041; GFX11-CU:       ; %bb.0: ; %entry
8042; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8043; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8044; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
8045; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
8046; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8047; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8048; GFX11-CU-NEXT:    s_endpgm
8049    i32 addrspace(3)* %out, i32 %in, i32 %old) {
8050entry:
8051  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
8052  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
8053  ret void
8054}
8055
8056define amdgpu_kernel void @local_singlethread_one_as_release_monotonic_cmpxchg(
8057; GFX6-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
8058; GFX6:       ; %bb.0: ; %entry
8059; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
8060; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
8061; GFX6-NEXT:    s_mov_b32 m0, -1
8062; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8063; GFX6-NEXT:    v_mov_b32_e32 v0, s2
8064; GFX6-NEXT:    v_mov_b32_e32 v1, s1
8065; GFX6-NEXT:    v_mov_b32_e32 v2, s0
8066; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8067; GFX6-NEXT:    s_endpgm
8068;
8069; GFX7-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
8070; GFX7:       ; %bb.0: ; %entry
8071; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8072; GFX7-NEXT:    s_mov_b32 m0, -1
8073; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8074; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8075; GFX7-NEXT:    v_mov_b32_e32 v1, s2
8076; GFX7-NEXT:    v_mov_b32_e32 v2, s1
8077; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8078; GFX7-NEXT:    s_endpgm
8079;
8080; GFX10-WGP-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
8081; GFX10-WGP:       ; %bb.0: ; %entry
8082; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8083; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8084; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8085; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8086; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
8087; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8088; GFX10-WGP-NEXT:    s_endpgm
8089;
8090; GFX10-CU-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
8091; GFX10-CU:       ; %bb.0: ; %entry
8092; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8093; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8094; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8095; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8096; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8097; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8098; GFX10-CU-NEXT:    s_endpgm
8099;
8100; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
8101; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8102; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8103; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
8104; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8105; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8106; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
8107; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
8108; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8109; SKIP-CACHE-INV-NEXT:    s_endpgm
8110;
8111; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
8112; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8113; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8114; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8115; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8116; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8117; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8118; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8119; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8120;
8121; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
8122; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8123; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8124; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8125; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8126; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8127; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8128; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8129; GFX90A-TGSPLIT-NEXT:    s_endpgm
8130;
8131; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
8132; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8133; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8134; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8135; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8136; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8137; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8138; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8139; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8140;
8141; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
8142; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8143; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8144; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8145; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8146; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8147; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8148; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8149; GFX940-TGSPLIT-NEXT:    s_endpgm
8150;
8151; GFX11-WGP-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
8152; GFX11-WGP:       ; %bb.0: ; %entry
8153; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8154; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8155; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
8156; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
8157; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
8158; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8159; GFX11-WGP-NEXT:    s_endpgm
8160;
8161; GFX11-CU-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
8162; GFX11-CU:       ; %bb.0: ; %entry
8163; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8164; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8165; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
8166; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
8167; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8168; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8169; GFX11-CU-NEXT:    s_endpgm
8170    i32 addrspace(3)* %out, i32 %in, i32 %old) {
8171entry:
8172  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
8173  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
8174  ret void
8175}
8176
8177define amdgpu_kernel void @local_singlethread_one_as_acq_rel_monotonic_cmpxchg(
8178; GFX6-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8179; GFX6:       ; %bb.0: ; %entry
8180; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
8181; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
8182; GFX6-NEXT:    s_mov_b32 m0, -1
8183; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8184; GFX6-NEXT:    v_mov_b32_e32 v0, s2
8185; GFX6-NEXT:    v_mov_b32_e32 v1, s1
8186; GFX6-NEXT:    v_mov_b32_e32 v2, s0
8187; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8188; GFX6-NEXT:    s_endpgm
8189;
8190; GFX7-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8191; GFX7:       ; %bb.0: ; %entry
8192; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8193; GFX7-NEXT:    s_mov_b32 m0, -1
8194; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8195; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8196; GFX7-NEXT:    v_mov_b32_e32 v1, s2
8197; GFX7-NEXT:    v_mov_b32_e32 v2, s1
8198; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8199; GFX7-NEXT:    s_endpgm
8200;
8201; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8202; GFX10-WGP:       ; %bb.0: ; %entry
8203; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8204; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8205; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8206; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8207; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
8208; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8209; GFX10-WGP-NEXT:    s_endpgm
8210;
8211; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8212; GFX10-CU:       ; %bb.0: ; %entry
8213; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8214; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8215; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8216; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8217; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8218; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8219; GFX10-CU-NEXT:    s_endpgm
8220;
8221; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8222; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8223; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8224; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
8225; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8226; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8227; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
8228; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
8229; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8230; SKIP-CACHE-INV-NEXT:    s_endpgm
8231;
8232; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8233; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8234; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8235; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8236; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8237; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8238; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8239; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8240; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8241;
8242; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8243; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8244; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8245; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8246; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8247; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8248; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8249; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8250; GFX90A-TGSPLIT-NEXT:    s_endpgm
8251;
8252; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8253; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8254; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8255; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8256; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8257; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8258; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8259; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8260; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8261;
8262; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8263; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8264; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8265; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8266; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8267; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8268; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8269; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8270; GFX940-TGSPLIT-NEXT:    s_endpgm
8271;
8272; GFX11-WGP-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8273; GFX11-WGP:       ; %bb.0: ; %entry
8274; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8275; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8276; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
8277; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
8278; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
8279; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8280; GFX11-WGP-NEXT:    s_endpgm
8281;
8282; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8283; GFX11-CU:       ; %bb.0: ; %entry
8284; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8285; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8286; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
8287; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
8288; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8289; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8290; GFX11-CU-NEXT:    s_endpgm
8291    i32 addrspace(3)* %out, i32 %in, i32 %old) {
8292entry:
8293  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
8294  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
8295  ret void
8296}
8297
8298define amdgpu_kernel void @local_singlethread_one_as_seq_cst_monotonic_cmpxchg(
8299; GFX6-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8300; GFX6:       ; %bb.0: ; %entry
8301; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
8302; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
8303; GFX6-NEXT:    s_mov_b32 m0, -1
8304; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8305; GFX6-NEXT:    v_mov_b32_e32 v0, s2
8306; GFX6-NEXT:    v_mov_b32_e32 v1, s1
8307; GFX6-NEXT:    v_mov_b32_e32 v2, s0
8308; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8309; GFX6-NEXT:    s_endpgm
8310;
8311; GFX7-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8312; GFX7:       ; %bb.0: ; %entry
8313; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8314; GFX7-NEXT:    s_mov_b32 m0, -1
8315; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8316; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8317; GFX7-NEXT:    v_mov_b32_e32 v1, s2
8318; GFX7-NEXT:    v_mov_b32_e32 v2, s1
8319; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8320; GFX7-NEXT:    s_endpgm
8321;
8322; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8323; GFX10-WGP:       ; %bb.0: ; %entry
8324; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8325; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8326; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8327; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8328; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
8329; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8330; GFX10-WGP-NEXT:    s_endpgm
8331;
8332; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8333; GFX10-CU:       ; %bb.0: ; %entry
8334; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8335; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8336; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8337; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8338; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8339; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8340; GFX10-CU-NEXT:    s_endpgm
8341;
8342; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8343; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8344; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8345; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
8346; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8347; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8348; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
8349; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
8350; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8351; SKIP-CACHE-INV-NEXT:    s_endpgm
8352;
8353; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8354; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8355; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8356; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8357; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8358; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8359; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8360; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8361; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8362;
8363; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8364; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8365; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8366; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8367; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8368; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8369; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8370; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8371; GFX90A-TGSPLIT-NEXT:    s_endpgm
8372;
8373; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8374; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8375; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8376; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8377; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8378; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8379; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8380; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8381; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8382;
8383; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8384; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8385; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8386; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8387; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8388; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8389; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8390; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8391; GFX940-TGSPLIT-NEXT:    s_endpgm
8392;
8393; GFX11-WGP-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8394; GFX11-WGP:       ; %bb.0: ; %entry
8395; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8396; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8397; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
8398; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
8399; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
8400; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8401; GFX11-WGP-NEXT:    s_endpgm
8402;
8403; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8404; GFX11-CU:       ; %bb.0: ; %entry
8405; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8406; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8407; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
8408; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
8409; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8410; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8411; GFX11-CU-NEXT:    s_endpgm
8412    i32 addrspace(3)* %out, i32 %in, i32 %old) {
8413entry:
8414  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
8415  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
8416  ret void
8417}
8418
8419define amdgpu_kernel void @local_singlethread_one_as_monotonic_acquire_cmpxchg(
8420; GFX6-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8421; GFX6:       ; %bb.0: ; %entry
8422; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
8423; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
8424; GFX6-NEXT:    s_mov_b32 m0, -1
8425; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8426; GFX6-NEXT:    v_mov_b32_e32 v0, s2
8427; GFX6-NEXT:    v_mov_b32_e32 v1, s1
8428; GFX6-NEXT:    v_mov_b32_e32 v2, s0
8429; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8430; GFX6-NEXT:    s_endpgm
8431;
8432; GFX7-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8433; GFX7:       ; %bb.0: ; %entry
8434; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8435; GFX7-NEXT:    s_mov_b32 m0, -1
8436; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8437; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8438; GFX7-NEXT:    v_mov_b32_e32 v1, s2
8439; GFX7-NEXT:    v_mov_b32_e32 v2, s1
8440; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8441; GFX7-NEXT:    s_endpgm
8442;
8443; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8444; GFX10-WGP:       ; %bb.0: ; %entry
8445; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8446; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8447; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8448; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8449; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
8450; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8451; GFX10-WGP-NEXT:    s_endpgm
8452;
8453; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8454; GFX10-CU:       ; %bb.0: ; %entry
8455; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8456; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8457; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8458; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8459; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8460; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8461; GFX10-CU-NEXT:    s_endpgm
8462;
8463; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8464; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8465; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8466; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
8467; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8468; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8469; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
8470; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
8471; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8472; SKIP-CACHE-INV-NEXT:    s_endpgm
8473;
8474; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8475; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8476; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8477; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8478; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8479; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8480; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8481; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8482; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8483;
8484; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8485; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8486; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8487; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8488; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8489; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8490; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8491; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8492; GFX90A-TGSPLIT-NEXT:    s_endpgm
8493;
8494; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8495; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8496; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8497; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8498; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8499; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8500; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8501; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8502; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8503;
8504; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8505; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8506; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8507; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8508; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8509; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8510; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8511; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8512; GFX940-TGSPLIT-NEXT:    s_endpgm
8513;
8514; GFX11-WGP-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8515; GFX11-WGP:       ; %bb.0: ; %entry
8516; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8517; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8518; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
8519; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
8520; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
8521; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8522; GFX11-WGP-NEXT:    s_endpgm
8523;
8524; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8525; GFX11-CU:       ; %bb.0: ; %entry
8526; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8527; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8528; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
8529; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
8530; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8531; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8532; GFX11-CU-NEXT:    s_endpgm
8533    i32 addrspace(3)* %out, i32 %in, i32 %old) {
8534entry:
8535  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
8536  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic acquire
8537  ret void
8538}
8539
8540define amdgpu_kernel void @local_singlethread_one_as_acquire_acquire_cmpxchg(
8541; GFX6-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8542; GFX6:       ; %bb.0: ; %entry
8543; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
8544; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
8545; GFX6-NEXT:    s_mov_b32 m0, -1
8546; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8547; GFX6-NEXT:    v_mov_b32_e32 v0, s2
8548; GFX6-NEXT:    v_mov_b32_e32 v1, s1
8549; GFX6-NEXT:    v_mov_b32_e32 v2, s0
8550; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8551; GFX6-NEXT:    s_endpgm
8552;
8553; GFX7-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8554; GFX7:       ; %bb.0: ; %entry
8555; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8556; GFX7-NEXT:    s_mov_b32 m0, -1
8557; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8558; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8559; GFX7-NEXT:    v_mov_b32_e32 v1, s2
8560; GFX7-NEXT:    v_mov_b32_e32 v2, s1
8561; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8562; GFX7-NEXT:    s_endpgm
8563;
8564; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8565; GFX10-WGP:       ; %bb.0: ; %entry
8566; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8567; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8568; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8569; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8570; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
8571; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8572; GFX10-WGP-NEXT:    s_endpgm
8573;
8574; GFX10-CU-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8575; GFX10-CU:       ; %bb.0: ; %entry
8576; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8577; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8578; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8579; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8580; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8581; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8582; GFX10-CU-NEXT:    s_endpgm
8583;
8584; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8585; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8586; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8587; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
8588; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8589; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8590; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
8591; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
8592; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8593; SKIP-CACHE-INV-NEXT:    s_endpgm
8594;
8595; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8596; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8597; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8598; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8599; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8600; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8601; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8602; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8603; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8604;
8605; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8606; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8607; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8608; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8609; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8610; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8611; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8612; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8613; GFX90A-TGSPLIT-NEXT:    s_endpgm
8614;
8615; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8616; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8617; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8618; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8619; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8620; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8621; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8622; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8623; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8624;
8625; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8626; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8627; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8628; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8629; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8630; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8631; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8632; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8633; GFX940-TGSPLIT-NEXT:    s_endpgm
8634;
8635; GFX11-WGP-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8636; GFX11-WGP:       ; %bb.0: ; %entry
8637; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8638; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8639; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
8640; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
8641; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
8642; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8643; GFX11-WGP-NEXT:    s_endpgm
8644;
8645; GFX11-CU-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8646; GFX11-CU:       ; %bb.0: ; %entry
8647; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8648; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8649; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
8650; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
8651; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8652; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8653; GFX11-CU-NEXT:    s_endpgm
8654    i32 addrspace(3)* %out, i32 %in, i32 %old) {
8655entry:
8656  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
8657  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
8658  ret void
8659}
8660
8661define amdgpu_kernel void @local_singlethread_one_as_release_acquire_cmpxchg(
8662; GFX6-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8663; GFX6:       ; %bb.0: ; %entry
8664; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
8665; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
8666; GFX6-NEXT:    s_mov_b32 m0, -1
8667; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8668; GFX6-NEXT:    v_mov_b32_e32 v0, s2
8669; GFX6-NEXT:    v_mov_b32_e32 v1, s1
8670; GFX6-NEXT:    v_mov_b32_e32 v2, s0
8671; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8672; GFX6-NEXT:    s_endpgm
8673;
8674; GFX7-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8675; GFX7:       ; %bb.0: ; %entry
8676; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8677; GFX7-NEXT:    s_mov_b32 m0, -1
8678; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8679; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8680; GFX7-NEXT:    v_mov_b32_e32 v1, s2
8681; GFX7-NEXT:    v_mov_b32_e32 v2, s1
8682; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8683; GFX7-NEXT:    s_endpgm
8684;
8685; GFX10-WGP-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8686; GFX10-WGP:       ; %bb.0: ; %entry
8687; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8688; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8689; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8690; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8691; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
8692; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8693; GFX10-WGP-NEXT:    s_endpgm
8694;
8695; GFX10-CU-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8696; GFX10-CU:       ; %bb.0: ; %entry
8697; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8698; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8699; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8700; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8701; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8702; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8703; GFX10-CU-NEXT:    s_endpgm
8704;
8705; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8706; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8707; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8708; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
8709; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8710; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8711; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
8712; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
8713; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8714; SKIP-CACHE-INV-NEXT:    s_endpgm
8715;
8716; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8717; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8718; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8719; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8720; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8721; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8722; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8723; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8724; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8725;
8726; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8727; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8728; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8729; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8730; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8731; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8732; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8733; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8734; GFX90A-TGSPLIT-NEXT:    s_endpgm
8735;
8736; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8737; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8738; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8739; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8740; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8741; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8742; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8743; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8744; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8745;
8746; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8747; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8748; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8749; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8750; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8751; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8752; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8753; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8754; GFX940-TGSPLIT-NEXT:    s_endpgm
8755;
8756; GFX11-WGP-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8757; GFX11-WGP:       ; %bb.0: ; %entry
8758; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8759; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8760; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
8761; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
8762; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
8763; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8764; GFX11-WGP-NEXT:    s_endpgm
8765;
8766; GFX11-CU-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8767; GFX11-CU:       ; %bb.0: ; %entry
8768; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8769; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8770; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
8771; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
8772; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8773; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8774; GFX11-CU-NEXT:    s_endpgm
8775    i32 addrspace(3)* %out, i32 %in, i32 %old) {
8776entry:
8777  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
8778  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
8779  ret void
8780}
8781
8782define amdgpu_kernel void @local_singlethread_one_as_acq_rel_acquire_cmpxchg(
8783; GFX6-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8784; GFX6:       ; %bb.0: ; %entry
8785; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
8786; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
8787; GFX6-NEXT:    s_mov_b32 m0, -1
8788; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8789; GFX6-NEXT:    v_mov_b32_e32 v0, s2
8790; GFX6-NEXT:    v_mov_b32_e32 v1, s1
8791; GFX6-NEXT:    v_mov_b32_e32 v2, s0
8792; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8793; GFX6-NEXT:    s_endpgm
8794;
8795; GFX7-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8796; GFX7:       ; %bb.0: ; %entry
8797; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8798; GFX7-NEXT:    s_mov_b32 m0, -1
8799; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8800; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8801; GFX7-NEXT:    v_mov_b32_e32 v1, s2
8802; GFX7-NEXT:    v_mov_b32_e32 v2, s1
8803; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8804; GFX7-NEXT:    s_endpgm
8805;
8806; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8807; GFX10-WGP:       ; %bb.0: ; %entry
8808; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8809; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8810; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8811; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8812; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
8813; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8814; GFX10-WGP-NEXT:    s_endpgm
8815;
8816; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8817; GFX10-CU:       ; %bb.0: ; %entry
8818; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8819; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8820; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8821; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8822; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8823; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8824; GFX10-CU-NEXT:    s_endpgm
8825;
8826; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8827; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8828; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8829; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
8830; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8831; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8832; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
8833; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
8834; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8835; SKIP-CACHE-INV-NEXT:    s_endpgm
8836;
8837; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8838; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8839; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8840; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8841; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8842; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8843; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8844; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8845; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8846;
8847; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8848; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8849; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8850; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8851; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8852; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8853; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8854; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8855; GFX90A-TGSPLIT-NEXT:    s_endpgm
8856;
8857; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8858; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8859; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8860; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8861; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8862; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8863; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8864; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8865; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8866;
8867; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8868; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8869; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8870; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8871; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8872; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8873; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8874; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8875; GFX940-TGSPLIT-NEXT:    s_endpgm
8876;
8877; GFX11-WGP-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8878; GFX11-WGP:       ; %bb.0: ; %entry
8879; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8880; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8881; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
8882; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
8883; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
8884; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8885; GFX11-WGP-NEXT:    s_endpgm
8886;
8887; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8888; GFX11-CU:       ; %bb.0: ; %entry
8889; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8890; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8891; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
8892; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
8893; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8894; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8895; GFX11-CU-NEXT:    s_endpgm
8896    i32 addrspace(3)* %out, i32 %in, i32 %old) {
8897entry:
8898  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
8899  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
8900  ret void
8901}
8902
8903define amdgpu_kernel void @local_singlethread_one_as_seq_cst_acquire_cmpxchg(
8904; GFX6-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8905; GFX6:       ; %bb.0: ; %entry
8906; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
8907; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
8908; GFX6-NEXT:    s_mov_b32 m0, -1
8909; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8910; GFX6-NEXT:    v_mov_b32_e32 v0, s2
8911; GFX6-NEXT:    v_mov_b32_e32 v1, s1
8912; GFX6-NEXT:    v_mov_b32_e32 v2, s0
8913; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8914; GFX6-NEXT:    s_endpgm
8915;
8916; GFX7-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8917; GFX7:       ; %bb.0: ; %entry
8918; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8919; GFX7-NEXT:    s_mov_b32 m0, -1
8920; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8921; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8922; GFX7-NEXT:    v_mov_b32_e32 v1, s2
8923; GFX7-NEXT:    v_mov_b32_e32 v2, s1
8924; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8925; GFX7-NEXT:    s_endpgm
8926;
8927; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8928; GFX10-WGP:       ; %bb.0: ; %entry
8929; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8930; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8931; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8932; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8933; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
8934; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8935; GFX10-WGP-NEXT:    s_endpgm
8936;
8937; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8938; GFX10-CU:       ; %bb.0: ; %entry
8939; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8940; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8941; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8942; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8943; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8944; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8945; GFX10-CU-NEXT:    s_endpgm
8946;
8947; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8948; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8949; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8950; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
8951; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8952; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8953; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
8954; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
8955; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8956; SKIP-CACHE-INV-NEXT:    s_endpgm
8957;
8958; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8959; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8960; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8961; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8962; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8963; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8964; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8965; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8966; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8967;
8968; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8969; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8970; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8971; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8972; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8973; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8974; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8975; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8976; GFX90A-TGSPLIT-NEXT:    s_endpgm
8977;
8978; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8979; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8980; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8981; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8982; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8983; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8984; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8985; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8986; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8987;
8988; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8989; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8990; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8991; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8992; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8993; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8994; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8995; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8996; GFX940-TGSPLIT-NEXT:    s_endpgm
8997;
8998; GFX11-WGP-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8999; GFX11-WGP:       ; %bb.0: ; %entry
9000; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9001; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9002; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
9003; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
9004; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
9005; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9006; GFX11-WGP-NEXT:    s_endpgm
9007;
9008; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
9009; GFX11-CU:       ; %bb.0: ; %entry
9010; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9011; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9012; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
9013; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
9014; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9015; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9016; GFX11-CU-NEXT:    s_endpgm
9017    i32 addrspace(3)* %out, i32 %in, i32 %old) {
9018entry:
9019  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
9020  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
9021  ret void
9022}
9023
9024define amdgpu_kernel void @local_singlethread_one_as_monotonic_seq_cst_cmpxchg(
9025; GFX6-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9026; GFX6:       ; %bb.0: ; %entry
9027; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
9028; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
9029; GFX6-NEXT:    s_mov_b32 m0, -1
9030; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9031; GFX6-NEXT:    v_mov_b32_e32 v0, s2
9032; GFX6-NEXT:    v_mov_b32_e32 v1, s1
9033; GFX6-NEXT:    v_mov_b32_e32 v2, s0
9034; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9035; GFX6-NEXT:    s_endpgm
9036;
9037; GFX7-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9038; GFX7:       ; %bb.0: ; %entry
9039; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9040; GFX7-NEXT:    s_mov_b32 m0, -1
9041; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9042; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9043; GFX7-NEXT:    v_mov_b32_e32 v1, s2
9044; GFX7-NEXT:    v_mov_b32_e32 v2, s1
9045; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9046; GFX7-NEXT:    s_endpgm
9047;
9048; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9049; GFX10-WGP:       ; %bb.0: ; %entry
9050; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9051; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9052; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9053; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
9054; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
9055; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9056; GFX10-WGP-NEXT:    s_endpgm
9057;
9058; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9059; GFX10-CU:       ; %bb.0: ; %entry
9060; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9061; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9062; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9063; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9064; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9065; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9066; GFX10-CU-NEXT:    s_endpgm
9067;
9068; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9069; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9070; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9071; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
9072; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9073; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9074; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
9075; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
9076; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9077; SKIP-CACHE-INV-NEXT:    s_endpgm
9078;
9079; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9080; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9081; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9082; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9083; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9084; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9085; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9086; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9087; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9088;
9089; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9090; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9091; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9092; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9093; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9094; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9095; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9096; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9097; GFX90A-TGSPLIT-NEXT:    s_endpgm
9098;
9099; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9100; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9101; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9102; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9103; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9104; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9105; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9106; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9107; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9108;
9109; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9110; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9111; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9112; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9113; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9114; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9115; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9116; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9117; GFX940-TGSPLIT-NEXT:    s_endpgm
9118;
9119; GFX11-WGP-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9120; GFX11-WGP:       ; %bb.0: ; %entry
9121; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9122; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9123; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
9124; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
9125; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
9126; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9127; GFX11-WGP-NEXT:    s_endpgm
9128;
9129; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9130; GFX11-CU:       ; %bb.0: ; %entry
9131; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9132; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9133; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
9134; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
9135; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9136; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9137; GFX11-CU-NEXT:    s_endpgm
9138    i32 addrspace(3)* %out, i32 %in, i32 %old) {
9139entry:
9140  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
9141  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic seq_cst
9142  ret void
9143}
9144
9145define amdgpu_kernel void @local_singlethread_one_as_acquire_seq_cst_cmpxchg(
9146; GFX6-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9147; GFX6:       ; %bb.0: ; %entry
9148; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
9149; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
9150; GFX6-NEXT:    s_mov_b32 m0, -1
9151; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9152; GFX6-NEXT:    v_mov_b32_e32 v0, s2
9153; GFX6-NEXT:    v_mov_b32_e32 v1, s1
9154; GFX6-NEXT:    v_mov_b32_e32 v2, s0
9155; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9156; GFX6-NEXT:    s_endpgm
9157;
9158; GFX7-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9159; GFX7:       ; %bb.0: ; %entry
9160; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9161; GFX7-NEXT:    s_mov_b32 m0, -1
9162; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9163; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9164; GFX7-NEXT:    v_mov_b32_e32 v1, s2
9165; GFX7-NEXT:    v_mov_b32_e32 v2, s1
9166; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9167; GFX7-NEXT:    s_endpgm
9168;
9169; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9170; GFX10-WGP:       ; %bb.0: ; %entry
9171; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9172; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9173; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9174; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
9175; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
9176; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9177; GFX10-WGP-NEXT:    s_endpgm
9178;
9179; GFX10-CU-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9180; GFX10-CU:       ; %bb.0: ; %entry
9181; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9182; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9183; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9184; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9185; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9186; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9187; GFX10-CU-NEXT:    s_endpgm
9188;
9189; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9190; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9191; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9192; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
9193; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9194; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9195; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
9196; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
9197; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9198; SKIP-CACHE-INV-NEXT:    s_endpgm
9199;
9200; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9201; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9202; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9203; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9204; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9205; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9206; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9207; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9208; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9209;
9210; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9211; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9212; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9213; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9214; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9215; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9216; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9217; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9218; GFX90A-TGSPLIT-NEXT:    s_endpgm
9219;
9220; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9221; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9222; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9223; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9224; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9225; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9226; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9227; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9228; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9229;
9230; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9231; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9232; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9233; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9234; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9235; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9236; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9237; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9238; GFX940-TGSPLIT-NEXT:    s_endpgm
9239;
9240; GFX11-WGP-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9241; GFX11-WGP:       ; %bb.0: ; %entry
9242; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9243; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9244; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
9245; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
9246; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
9247; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9248; GFX11-WGP-NEXT:    s_endpgm
9249;
9250; GFX11-CU-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9251; GFX11-CU:       ; %bb.0: ; %entry
9252; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9253; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9254; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
9255; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
9256; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9257; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9258; GFX11-CU-NEXT:    s_endpgm
9259    i32 addrspace(3)* %out, i32 %in, i32 %old) {
9260entry:
9261  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
9262  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire seq_cst
9263  ret void
9264}
9265
9266define amdgpu_kernel void @local_singlethread_one_as_release_seq_cst_cmpxchg(
9267; GFX6-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9268; GFX6:       ; %bb.0: ; %entry
9269; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
9270; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
9271; GFX6-NEXT:    s_mov_b32 m0, -1
9272; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9273; GFX6-NEXT:    v_mov_b32_e32 v0, s2
9274; GFX6-NEXT:    v_mov_b32_e32 v1, s1
9275; GFX6-NEXT:    v_mov_b32_e32 v2, s0
9276; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9277; GFX6-NEXT:    s_endpgm
9278;
9279; GFX7-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9280; GFX7:       ; %bb.0: ; %entry
9281; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9282; GFX7-NEXT:    s_mov_b32 m0, -1
9283; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9284; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9285; GFX7-NEXT:    v_mov_b32_e32 v1, s2
9286; GFX7-NEXT:    v_mov_b32_e32 v2, s1
9287; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9288; GFX7-NEXT:    s_endpgm
9289;
9290; GFX10-WGP-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9291; GFX10-WGP:       ; %bb.0: ; %entry
9292; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9293; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9294; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9295; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
9296; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
9297; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9298; GFX10-WGP-NEXT:    s_endpgm
9299;
9300; GFX10-CU-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9301; GFX10-CU:       ; %bb.0: ; %entry
9302; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9303; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9304; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9305; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9306; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9307; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9308; GFX10-CU-NEXT:    s_endpgm
9309;
9310; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9311; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9312; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9313; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
9314; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9315; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9316; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
9317; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
9318; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9319; SKIP-CACHE-INV-NEXT:    s_endpgm
9320;
9321; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9322; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9323; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9324; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9325; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9326; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9327; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9328; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9329; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9330;
9331; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9332; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9333; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9334; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9335; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9336; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9337; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9338; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9339; GFX90A-TGSPLIT-NEXT:    s_endpgm
9340;
9341; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9342; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9343; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9344; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9345; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9346; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9347; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9348; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9349; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9350;
9351; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9352; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9353; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9354; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9355; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9356; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9357; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9358; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9359; GFX940-TGSPLIT-NEXT:    s_endpgm
9360;
9361; GFX11-WGP-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9362; GFX11-WGP:       ; %bb.0: ; %entry
9363; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9364; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9365; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
9366; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
9367; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
9368; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9369; GFX11-WGP-NEXT:    s_endpgm
9370;
9371; GFX11-CU-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9372; GFX11-CU:       ; %bb.0: ; %entry
9373; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9374; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9375; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
9376; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
9377; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9378; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9379; GFX11-CU-NEXT:    s_endpgm
9380    i32 addrspace(3)* %out, i32 %in, i32 %old) {
9381entry:
9382  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
9383  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release seq_cst
9384  ret void
9385}
9386
9387define amdgpu_kernel void @local_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
9388; GFX6-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9389; GFX6:       ; %bb.0: ; %entry
9390; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
9391; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
9392; GFX6-NEXT:    s_mov_b32 m0, -1
9393; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9394; GFX6-NEXT:    v_mov_b32_e32 v0, s2
9395; GFX6-NEXT:    v_mov_b32_e32 v1, s1
9396; GFX6-NEXT:    v_mov_b32_e32 v2, s0
9397; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9398; GFX6-NEXT:    s_endpgm
9399;
9400; GFX7-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9401; GFX7:       ; %bb.0: ; %entry
9402; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9403; GFX7-NEXT:    s_mov_b32 m0, -1
9404; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9405; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9406; GFX7-NEXT:    v_mov_b32_e32 v1, s2
9407; GFX7-NEXT:    v_mov_b32_e32 v2, s1
9408; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9409; GFX7-NEXT:    s_endpgm
9410;
9411; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9412; GFX10-WGP:       ; %bb.0: ; %entry
9413; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9414; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9415; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9416; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
9417; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
9418; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9419; GFX10-WGP-NEXT:    s_endpgm
9420;
9421; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9422; GFX10-CU:       ; %bb.0: ; %entry
9423; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9424; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9425; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9426; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9427; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9428; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9429; GFX10-CU-NEXT:    s_endpgm
9430;
9431; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9432; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9433; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9434; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
9435; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9436; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9437; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
9438; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
9439; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9440; SKIP-CACHE-INV-NEXT:    s_endpgm
9441;
9442; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9443; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9444; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9445; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9446; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9447; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9448; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9449; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9450; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9451;
9452; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9453; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9454; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9455; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9456; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9457; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9458; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9459; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9460; GFX90A-TGSPLIT-NEXT:    s_endpgm
9461;
9462; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9463; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9464; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9465; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9466; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9467; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9468; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9469; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9470; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9471;
9472; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9473; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9474; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9475; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9476; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9477; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9478; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9479; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9480; GFX940-TGSPLIT-NEXT:    s_endpgm
9481;
9482; GFX11-WGP-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9483; GFX11-WGP:       ; %bb.0: ; %entry
9484; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9485; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9486; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
9487; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
9488; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
9489; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9490; GFX11-WGP-NEXT:    s_endpgm
9491;
9492; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9493; GFX11-CU:       ; %bb.0: ; %entry
9494; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9495; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9496; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
9497; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
9498; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9499; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9500; GFX11-CU-NEXT:    s_endpgm
9501    i32 addrspace(3)* %out, i32 %in, i32 %old) {
9502entry:
9503  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
9504  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel seq_cst
9505  ret void
9506}
9507
9508define amdgpu_kernel void @local_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
9509; GFX6-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9510; GFX6:       ; %bb.0: ; %entry
9511; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
9512; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
9513; GFX6-NEXT:    s_mov_b32 m0, -1
9514; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9515; GFX6-NEXT:    v_mov_b32_e32 v0, s2
9516; GFX6-NEXT:    v_mov_b32_e32 v1, s1
9517; GFX6-NEXT:    v_mov_b32_e32 v2, s0
9518; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9519; GFX6-NEXT:    s_endpgm
9520;
9521; GFX7-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9522; GFX7:       ; %bb.0: ; %entry
9523; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9524; GFX7-NEXT:    s_mov_b32 m0, -1
9525; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9526; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9527; GFX7-NEXT:    v_mov_b32_e32 v1, s2
9528; GFX7-NEXT:    v_mov_b32_e32 v2, s1
9529; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9530; GFX7-NEXT:    s_endpgm
9531;
9532; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9533; GFX10-WGP:       ; %bb.0: ; %entry
9534; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9535; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9536; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9537; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
9538; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
9539; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9540; GFX10-WGP-NEXT:    s_endpgm
9541;
9542; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9543; GFX10-CU:       ; %bb.0: ; %entry
9544; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9545; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9546; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9547; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9548; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9549; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9550; GFX10-CU-NEXT:    s_endpgm
9551;
9552; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9553; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9554; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9555; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
9556; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9557; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9558; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
9559; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
9560; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9561; SKIP-CACHE-INV-NEXT:    s_endpgm
9562;
9563; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9564; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9565; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9566; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9567; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9568; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9569; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9570; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9571; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9572;
9573; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9574; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9575; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9576; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9577; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9578; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9579; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9580; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9581; GFX90A-TGSPLIT-NEXT:    s_endpgm
9582;
9583; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9584; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9585; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9586; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9587; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9588; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9589; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9590; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9591; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9592;
9593; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9594; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9595; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9596; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9597; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9598; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9599; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9600; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9601; GFX940-TGSPLIT-NEXT:    s_endpgm
9602;
9603; GFX11-WGP-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9604; GFX11-WGP:       ; %bb.0: ; %entry
9605; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9606; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9607; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
9608; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
9609; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
9610; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9611; GFX11-WGP-NEXT:    s_endpgm
9612;
9613; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9614; GFX11-CU:       ; %bb.0: ; %entry
9615; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9616; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9617; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
9618; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
9619; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9620; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9621; GFX11-CU-NEXT:    s_endpgm
9622    i32 addrspace(3)* %out, i32 %in, i32 %old) {
9623entry:
9624  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
9625  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
9626  ret void
9627}
9628
9629define amdgpu_kernel void @local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg(
9630; GFX6-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9631; GFX6:       ; %bb.0: ; %entry
9632; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
9633; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
9634; GFX6-NEXT:    s_mov_b32 m0, -1
9635; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9636; GFX6-NEXT:    v_mov_b32_e32 v0, s2
9637; GFX6-NEXT:    v_mov_b32_e32 v1, s1
9638; GFX6-NEXT:    v_mov_b32_e32 v2, s0
9639; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9640; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9641; GFX6-NEXT:    ds_write_b32 v0, v1
9642; GFX6-NEXT:    s_endpgm
9643;
9644; GFX7-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9645; GFX7:       ; %bb.0: ; %entry
9646; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9647; GFX7-NEXT:    s_mov_b32 m0, -1
9648; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9649; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9650; GFX7-NEXT:    v_mov_b32_e32 v1, s2
9651; GFX7-NEXT:    v_mov_b32_e32 v2, s1
9652; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9653; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9654; GFX7-NEXT:    ds_write_b32 v0, v1
9655; GFX7-NEXT:    s_endpgm
9656;
9657; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9658; GFX10-WGP:       ; %bb.0: ; %entry
9659; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9660; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9661; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9662; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
9663; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
9664; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9665; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9666; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
9667; GFX10-WGP-NEXT:    s_endpgm
9668;
9669; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9670; GFX10-CU:       ; %bb.0: ; %entry
9671; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9672; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9673; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9674; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9675; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9676; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9677; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9678; GFX10-CU-NEXT:    ds_write_b32 v0, v1
9679; GFX10-CU-NEXT:    s_endpgm
9680;
9681; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9682; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9683; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9684; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
9685; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9686; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9687; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
9688; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
9689; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9690; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9691; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
9692; SKIP-CACHE-INV-NEXT:    s_endpgm
9693;
9694; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9695; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9696; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9697; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9698; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9699; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9700; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9701; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9702; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9703; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
9704; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9705;
9706; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9707; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9708; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9709; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9710; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9711; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9712; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9713; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9714; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9715; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
9716; GFX90A-TGSPLIT-NEXT:    s_endpgm
9717;
9718; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9719; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9720; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9721; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9722; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9723; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9724; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9725; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9726; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9727; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
9728; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9729;
9730; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9731; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9732; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9733; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9734; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9735; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9736; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9737; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9738; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9739; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
9740; GFX940-TGSPLIT-NEXT:    s_endpgm
9741;
9742; GFX11-WGP-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9743; GFX11-WGP:       ; %bb.0: ; %entry
9744; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9745; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9746; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
9747; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
9748; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
9749; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
9750; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9751; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
9752; GFX11-WGP-NEXT:    s_endpgm
9753;
9754; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9755; GFX11-CU:       ; %bb.0: ; %entry
9756; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9757; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9758; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
9759; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
9760; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9761; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
9762; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9763; GFX11-CU-NEXT:    ds_store_b32 v0, v1
9764; GFX11-CU-NEXT:    s_endpgm
9765    i32 addrspace(3)* %out, i32 %in, i32 %old) {
9766entry:
9767  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
9768  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
9769  %val0 = extractvalue { i32, i1 } %val, 0
9770  store i32 %val0, i32 addrspace(3)* %out, align 4
9771  ret void
9772}
9773
9774define amdgpu_kernel void @local_singlethread_one_as_acquire_monotonic_ret_cmpxchg(
9775; GFX6-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9776; GFX6:       ; %bb.0: ; %entry
9777; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
9778; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
9779; GFX6-NEXT:    s_mov_b32 m0, -1
9780; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9781; GFX6-NEXT:    v_mov_b32_e32 v0, s2
9782; GFX6-NEXT:    v_mov_b32_e32 v1, s1
9783; GFX6-NEXT:    v_mov_b32_e32 v2, s0
9784; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9785; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9786; GFX6-NEXT:    ds_write_b32 v0, v1
9787; GFX6-NEXT:    s_endpgm
9788;
9789; GFX7-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9790; GFX7:       ; %bb.0: ; %entry
9791; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9792; GFX7-NEXT:    s_mov_b32 m0, -1
9793; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9794; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9795; GFX7-NEXT:    v_mov_b32_e32 v1, s2
9796; GFX7-NEXT:    v_mov_b32_e32 v2, s1
9797; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9798; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9799; GFX7-NEXT:    ds_write_b32 v0, v1
9800; GFX7-NEXT:    s_endpgm
9801;
9802; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9803; GFX10-WGP:       ; %bb.0: ; %entry
9804; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9805; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9806; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9807; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
9808; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
9809; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9810; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9811; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
9812; GFX10-WGP-NEXT:    s_endpgm
9813;
9814; GFX10-CU-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9815; GFX10-CU:       ; %bb.0: ; %entry
9816; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9817; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9818; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9819; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9820; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9821; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9822; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9823; GFX10-CU-NEXT:    ds_write_b32 v0, v1
9824; GFX10-CU-NEXT:    s_endpgm
9825;
9826; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9827; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9828; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9829; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
9830; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9831; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9832; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
9833; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
9834; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9835; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9836; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
9837; SKIP-CACHE-INV-NEXT:    s_endpgm
9838;
9839; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9840; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9841; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9842; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9843; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9844; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9845; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9846; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9847; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9848; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
9849; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9850;
9851; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9852; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9853; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9854; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9855; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9856; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9857; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9858; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9859; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9860; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
9861; GFX90A-TGSPLIT-NEXT:    s_endpgm
9862;
9863; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9864; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9865; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9866; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9867; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9868; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9869; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9870; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9871; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9872; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
9873; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9874;
9875; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9876; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9877; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9878; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9879; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9880; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9881; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9882; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9883; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9884; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
9885; GFX940-TGSPLIT-NEXT:    s_endpgm
9886;
9887; GFX11-WGP-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9888; GFX11-WGP:       ; %bb.0: ; %entry
9889; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9890; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9891; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
9892; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
9893; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
9894; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
9895; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9896; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
9897; GFX11-WGP-NEXT:    s_endpgm
9898;
9899; GFX11-CU-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9900; GFX11-CU:       ; %bb.0: ; %entry
9901; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9902; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9903; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
9904; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
9905; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9906; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
9907; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9908; GFX11-CU-NEXT:    ds_store_b32 v0, v1
9909; GFX11-CU-NEXT:    s_endpgm
9910    i32 addrspace(3)* %out, i32 %in, i32 %old) {
9911entry:
9912  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
9913  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
9914  %val0 = extractvalue { i32, i1 } %val, 0
9915  store i32 %val0, i32 addrspace(3)* %out, align 4
9916  ret void
9917}
9918
9919define amdgpu_kernel void @local_singlethread_one_as_release_monotonic_ret_cmpxchg(
9920; GFX6-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
9921; GFX6:       ; %bb.0: ; %entry
9922; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
9923; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
9924; GFX6-NEXT:    s_mov_b32 m0, -1
9925; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9926; GFX6-NEXT:    v_mov_b32_e32 v0, s2
9927; GFX6-NEXT:    v_mov_b32_e32 v1, s1
9928; GFX6-NEXT:    v_mov_b32_e32 v2, s0
9929; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9930; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9931; GFX6-NEXT:    ds_write_b32 v0, v1
9932; GFX6-NEXT:    s_endpgm
9933;
9934; GFX7-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
9935; GFX7:       ; %bb.0: ; %entry
9936; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9937; GFX7-NEXT:    s_mov_b32 m0, -1
9938; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9939; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9940; GFX7-NEXT:    v_mov_b32_e32 v1, s2
9941; GFX7-NEXT:    v_mov_b32_e32 v2, s1
9942; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9943; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9944; GFX7-NEXT:    ds_write_b32 v0, v1
9945; GFX7-NEXT:    s_endpgm
9946;
9947; GFX10-WGP-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
9948; GFX10-WGP:       ; %bb.0: ; %entry
9949; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9950; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9951; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9952; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
9953; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
9954; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9955; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9956; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
9957; GFX10-WGP-NEXT:    s_endpgm
9958;
9959; GFX10-CU-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
9960; GFX10-CU:       ; %bb.0: ; %entry
9961; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9962; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9963; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9964; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9965; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9966; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9967; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9968; GFX10-CU-NEXT:    ds_write_b32 v0, v1
9969; GFX10-CU-NEXT:    s_endpgm
9970;
9971; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
9972; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9973; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9974; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
9975; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9976; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9977; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
9978; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
9979; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9980; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9981; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
9982; SKIP-CACHE-INV-NEXT:    s_endpgm
9983;
9984; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
9985; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9986; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9987; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9988; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9989; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9990; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9991; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9992; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9993; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
9994; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9995;
9996; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
9997; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9998; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9999; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10000; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10001; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10002; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10003; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10004; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10005; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10006; GFX90A-TGSPLIT-NEXT:    s_endpgm
10007;
10008; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
10009; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10010; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10011; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10012; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10013; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10014; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10015; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10016; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10017; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10018; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10019;
10020; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
10021; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10022; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10023; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10024; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10025; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10026; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10027; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10028; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10029; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10030; GFX940-TGSPLIT-NEXT:    s_endpgm
10031;
10032; GFX11-WGP-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
10033; GFX11-WGP:       ; %bb.0: ; %entry
10034; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10035; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10036; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
10037; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
10038; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
10039; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10040; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10041; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
10042; GFX11-WGP-NEXT:    s_endpgm
10043;
10044; GFX11-CU-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
10045; GFX11-CU:       ; %bb.0: ; %entry
10046; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10047; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10048; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
10049; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
10050; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10051; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10052; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10053; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10054; GFX11-CU-NEXT:    s_endpgm
10055    i32 addrspace(3)* %out, i32 %in, i32 %old) {
10056entry:
10057  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
10058  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
10059  %val0 = extractvalue { i32, i1 } %val, 0
10060  store i32 %val0, i32 addrspace(3)* %out, align 4
10061  ret void
10062}
10063
10064define amdgpu_kernel void @local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg(
10065; GFX6-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10066; GFX6:       ; %bb.0: ; %entry
10067; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
10068; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
10069; GFX6-NEXT:    s_mov_b32 m0, -1
10070; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10071; GFX6-NEXT:    v_mov_b32_e32 v0, s2
10072; GFX6-NEXT:    v_mov_b32_e32 v1, s1
10073; GFX6-NEXT:    v_mov_b32_e32 v2, s0
10074; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10075; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10076; GFX6-NEXT:    ds_write_b32 v0, v1
10077; GFX6-NEXT:    s_endpgm
10078;
10079; GFX7-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10080; GFX7:       ; %bb.0: ; %entry
10081; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10082; GFX7-NEXT:    s_mov_b32 m0, -1
10083; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10084; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10085; GFX7-NEXT:    v_mov_b32_e32 v1, s2
10086; GFX7-NEXT:    v_mov_b32_e32 v2, s1
10087; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10088; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10089; GFX7-NEXT:    ds_write_b32 v0, v1
10090; GFX7-NEXT:    s_endpgm
10091;
10092; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10093; GFX10-WGP:       ; %bb.0: ; %entry
10094; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10095; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10096; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10097; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
10098; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
10099; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10100; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10101; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
10102; GFX10-WGP-NEXT:    s_endpgm
10103;
10104; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10105; GFX10-CU:       ; %bb.0: ; %entry
10106; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10107; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10108; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10109; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10110; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10111; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10112; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10113; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10114; GFX10-CU-NEXT:    s_endpgm
10115;
10116; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10117; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10118; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10119; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
10120; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10121; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
10122; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
10123; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
10124; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10125; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10126; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
10127; SKIP-CACHE-INV-NEXT:    s_endpgm
10128;
10129; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10130; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10131; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10132; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10133; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10134; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10135; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10136; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10137; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10138; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10139; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10140;
10141; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10142; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10143; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10144; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10145; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10146; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10147; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10148; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10149; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10150; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10151; GFX90A-TGSPLIT-NEXT:    s_endpgm
10152;
10153; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10154; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10155; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10156; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10157; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10158; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10159; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10160; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10161; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10162; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10163; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10164;
10165; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10166; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10167; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10168; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10169; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10170; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10171; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10172; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10173; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10174; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10175; GFX940-TGSPLIT-NEXT:    s_endpgm
10176;
10177; GFX11-WGP-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10178; GFX11-WGP:       ; %bb.0: ; %entry
10179; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10180; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10181; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
10182; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
10183; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
10184; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10185; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10186; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
10187; GFX11-WGP-NEXT:    s_endpgm
10188;
10189; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10190; GFX11-CU:       ; %bb.0: ; %entry
10191; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10192; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10193; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
10194; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
10195; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10196; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10197; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10198; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10199; GFX11-CU-NEXT:    s_endpgm
10200    i32 addrspace(3)* %out, i32 %in, i32 %old) {
10201entry:
10202  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
10203  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
10204  %val0 = extractvalue { i32, i1 } %val, 0
10205  store i32 %val0, i32 addrspace(3)* %out, align 4
10206  ret void
10207}
10208
10209define amdgpu_kernel void @local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg(
10210; GFX6-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10211; GFX6:       ; %bb.0: ; %entry
10212; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
10213; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
10214; GFX6-NEXT:    s_mov_b32 m0, -1
10215; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10216; GFX6-NEXT:    v_mov_b32_e32 v0, s2
10217; GFX6-NEXT:    v_mov_b32_e32 v1, s1
10218; GFX6-NEXT:    v_mov_b32_e32 v2, s0
10219; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10220; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10221; GFX6-NEXT:    ds_write_b32 v0, v1
10222; GFX6-NEXT:    s_endpgm
10223;
10224; GFX7-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10225; GFX7:       ; %bb.0: ; %entry
10226; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10227; GFX7-NEXT:    s_mov_b32 m0, -1
10228; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10229; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10230; GFX7-NEXT:    v_mov_b32_e32 v1, s2
10231; GFX7-NEXT:    v_mov_b32_e32 v2, s1
10232; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10233; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10234; GFX7-NEXT:    ds_write_b32 v0, v1
10235; GFX7-NEXT:    s_endpgm
10236;
10237; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10238; GFX10-WGP:       ; %bb.0: ; %entry
10239; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10240; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10241; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10242; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
10243; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
10244; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10245; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10246; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
10247; GFX10-WGP-NEXT:    s_endpgm
10248;
10249; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10250; GFX10-CU:       ; %bb.0: ; %entry
10251; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10252; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10253; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10254; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10255; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10256; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10257; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10258; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10259; GFX10-CU-NEXT:    s_endpgm
10260;
10261; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10262; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10263; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10264; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
10265; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10266; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
10267; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
10268; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
10269; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10270; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10271; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
10272; SKIP-CACHE-INV-NEXT:    s_endpgm
10273;
10274; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10275; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10276; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10277; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10278; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10279; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10280; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10281; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10282; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10283; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10284; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10285;
10286; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10287; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10288; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10289; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10290; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10291; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10292; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10293; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10294; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10295; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10296; GFX90A-TGSPLIT-NEXT:    s_endpgm
10297;
10298; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10299; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10300; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10301; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10302; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10303; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10304; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10305; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10306; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10307; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10308; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10309;
10310; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10311; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10312; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10313; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10314; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10315; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10316; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10317; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10318; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10319; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10320; GFX940-TGSPLIT-NEXT:    s_endpgm
10321;
10322; GFX11-WGP-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10323; GFX11-WGP:       ; %bb.0: ; %entry
10324; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10325; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10326; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
10327; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
10328; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
10329; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10330; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10331; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
10332; GFX11-WGP-NEXT:    s_endpgm
10333;
10334; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10335; GFX11-CU:       ; %bb.0: ; %entry
10336; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10337; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10338; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
10339; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
10340; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10341; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10342; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10343; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10344; GFX11-CU-NEXT:    s_endpgm
10345    i32 addrspace(3)* %out, i32 %in, i32 %old) {
10346entry:
10347  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
10348  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
10349  %val0 = extractvalue { i32, i1 } %val, 0
10350  store i32 %val0, i32 addrspace(3)* %out, align 4
10351  ret void
10352}
10353
10354define amdgpu_kernel void @local_singlethread_one_as_monotonic_acquire_ret_cmpxchg(
10355; GFX6-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10356; GFX6:       ; %bb.0: ; %entry
10357; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
10358; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
10359; GFX6-NEXT:    s_mov_b32 m0, -1
10360; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10361; GFX6-NEXT:    v_mov_b32_e32 v0, s2
10362; GFX6-NEXT:    v_mov_b32_e32 v1, s1
10363; GFX6-NEXT:    v_mov_b32_e32 v2, s0
10364; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10365; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10366; GFX6-NEXT:    ds_write_b32 v0, v1
10367; GFX6-NEXT:    s_endpgm
10368;
10369; GFX7-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10370; GFX7:       ; %bb.0: ; %entry
10371; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10372; GFX7-NEXT:    s_mov_b32 m0, -1
10373; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10374; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10375; GFX7-NEXT:    v_mov_b32_e32 v1, s2
10376; GFX7-NEXT:    v_mov_b32_e32 v2, s1
10377; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10378; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10379; GFX7-NEXT:    ds_write_b32 v0, v1
10380; GFX7-NEXT:    s_endpgm
10381;
10382; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10383; GFX10-WGP:       ; %bb.0: ; %entry
10384; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10385; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10386; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10387; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
10388; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
10389; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10390; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10391; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
10392; GFX10-WGP-NEXT:    s_endpgm
10393;
10394; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10395; GFX10-CU:       ; %bb.0: ; %entry
10396; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10397; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10398; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10399; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10400; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10401; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10402; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10403; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10404; GFX10-CU-NEXT:    s_endpgm
10405;
10406; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10407; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10408; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10409; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
10410; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10411; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
10412; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
10413; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
10414; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10415; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10416; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
10417; SKIP-CACHE-INV-NEXT:    s_endpgm
10418;
10419; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10420; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10421; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10422; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10423; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10424; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10425; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10426; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10427; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10428; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10429; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10430;
10431; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10432; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10433; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10434; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10435; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10436; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10437; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10438; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10439; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10440; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10441; GFX90A-TGSPLIT-NEXT:    s_endpgm
10442;
10443; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10444; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10445; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10446; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10447; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10448; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10449; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10450; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10451; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10452; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10453; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10454;
10455; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10456; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10457; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10458; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10459; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10460; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10461; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10462; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10463; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10464; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10465; GFX940-TGSPLIT-NEXT:    s_endpgm
10466;
10467; GFX11-WGP-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10468; GFX11-WGP:       ; %bb.0: ; %entry
10469; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10470; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10471; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
10472; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
10473; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
10474; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10475; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10476; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
10477; GFX11-WGP-NEXT:    s_endpgm
10478;
10479; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10480; GFX11-CU:       ; %bb.0: ; %entry
10481; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10482; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10483; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
10484; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
10485; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10486; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10487; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10488; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10489; GFX11-CU-NEXT:    s_endpgm
10490    i32 addrspace(3)* %out, i32 %in, i32 %old) {
10491entry:
10492  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
10493  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic acquire
10494  %val0 = extractvalue { i32, i1 } %val, 0
10495  store i32 %val0, i32 addrspace(3)* %out, align 4
10496  ret void
10497}
10498
10499define amdgpu_kernel void @local_singlethread_one_as_acquire_acquire_ret_cmpxchg(
10500; GFX6-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10501; GFX6:       ; %bb.0: ; %entry
10502; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
10503; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
10504; GFX6-NEXT:    s_mov_b32 m0, -1
10505; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10506; GFX6-NEXT:    v_mov_b32_e32 v0, s2
10507; GFX6-NEXT:    v_mov_b32_e32 v1, s1
10508; GFX6-NEXT:    v_mov_b32_e32 v2, s0
10509; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10510; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10511; GFX6-NEXT:    ds_write_b32 v0, v1
10512; GFX6-NEXT:    s_endpgm
10513;
10514; GFX7-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10515; GFX7:       ; %bb.0: ; %entry
10516; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10517; GFX7-NEXT:    s_mov_b32 m0, -1
10518; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10519; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10520; GFX7-NEXT:    v_mov_b32_e32 v1, s2
10521; GFX7-NEXT:    v_mov_b32_e32 v2, s1
10522; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10523; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10524; GFX7-NEXT:    ds_write_b32 v0, v1
10525; GFX7-NEXT:    s_endpgm
10526;
10527; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10528; GFX10-WGP:       ; %bb.0: ; %entry
10529; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10530; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10531; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10532; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
10533; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
10534; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10535; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10536; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
10537; GFX10-WGP-NEXT:    s_endpgm
10538;
10539; GFX10-CU-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10540; GFX10-CU:       ; %bb.0: ; %entry
10541; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10542; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10543; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10544; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10545; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10546; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10547; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10548; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10549; GFX10-CU-NEXT:    s_endpgm
10550;
10551; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10552; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10553; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10554; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
10555; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10556; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
10557; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
10558; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
10559; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10560; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10561; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
10562; SKIP-CACHE-INV-NEXT:    s_endpgm
10563;
10564; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10565; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10566; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10567; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10568; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10569; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10570; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10571; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10572; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10573; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10574; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10575;
10576; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10577; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10578; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10579; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10580; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10581; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10582; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10583; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10584; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10585; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10586; GFX90A-TGSPLIT-NEXT:    s_endpgm
10587;
10588; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10589; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10590; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10591; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10592; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10593; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10594; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10595; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10596; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10597; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10598; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10599;
10600; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10601; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10602; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10603; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10604; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10605; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10606; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10607; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10608; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10609; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10610; GFX940-TGSPLIT-NEXT:    s_endpgm
10611;
10612; GFX11-WGP-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10613; GFX11-WGP:       ; %bb.0: ; %entry
10614; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10615; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10616; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
10617; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
10618; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
10619; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10620; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10621; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
10622; GFX11-WGP-NEXT:    s_endpgm
10623;
10624; GFX11-CU-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10625; GFX11-CU:       ; %bb.0: ; %entry
10626; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10627; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10628; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
10629; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
10630; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10631; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10632; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10633; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10634; GFX11-CU-NEXT:    s_endpgm
10635    i32 addrspace(3)* %out, i32 %in, i32 %old) {
10636entry:
10637  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
10638  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
10639  %val0 = extractvalue { i32, i1 } %val, 0
10640  store i32 %val0, i32 addrspace(3)* %out, align 4
10641  ret void
10642}
10643
10644define amdgpu_kernel void @local_singlethread_one_as_release_acquire_ret_cmpxchg(
10645; GFX6-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10646; GFX6:       ; %bb.0: ; %entry
10647; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
10648; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
10649; GFX6-NEXT:    s_mov_b32 m0, -1
10650; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10651; GFX6-NEXT:    v_mov_b32_e32 v0, s2
10652; GFX6-NEXT:    v_mov_b32_e32 v1, s1
10653; GFX6-NEXT:    v_mov_b32_e32 v2, s0
10654; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10655; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10656; GFX6-NEXT:    ds_write_b32 v0, v1
10657; GFX6-NEXT:    s_endpgm
10658;
10659; GFX7-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10660; GFX7:       ; %bb.0: ; %entry
10661; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10662; GFX7-NEXT:    s_mov_b32 m0, -1
10663; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10664; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10665; GFX7-NEXT:    v_mov_b32_e32 v1, s2
10666; GFX7-NEXT:    v_mov_b32_e32 v2, s1
10667; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10668; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10669; GFX7-NEXT:    ds_write_b32 v0, v1
10670; GFX7-NEXT:    s_endpgm
10671;
10672; GFX10-WGP-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10673; GFX10-WGP:       ; %bb.0: ; %entry
10674; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10675; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10676; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10677; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
10678; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
10679; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10680; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10681; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
10682; GFX10-WGP-NEXT:    s_endpgm
10683;
10684; GFX10-CU-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10685; GFX10-CU:       ; %bb.0: ; %entry
10686; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10687; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10688; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10689; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10690; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10691; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10692; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10693; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10694; GFX10-CU-NEXT:    s_endpgm
10695;
10696; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10697; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10698; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10699; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
10700; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10701; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
10702; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
10703; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
10704; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10705; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10706; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
10707; SKIP-CACHE-INV-NEXT:    s_endpgm
10708;
10709; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10710; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10711; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10712; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10713; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10714; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10715; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10716; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10717; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10718; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10719; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10720;
10721; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10722; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10723; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10724; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10725; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10726; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10727; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10728; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10729; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10730; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10731; GFX90A-TGSPLIT-NEXT:    s_endpgm
10732;
10733; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10734; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10735; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10736; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10737; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10738; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10739; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10740; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10741; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10742; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10743; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10744;
10745; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10746; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10747; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10748; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10749; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10750; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10751; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10752; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10753; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10754; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10755; GFX940-TGSPLIT-NEXT:    s_endpgm
10756;
10757; GFX11-WGP-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10758; GFX11-WGP:       ; %bb.0: ; %entry
10759; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10760; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10761; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
10762; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
10763; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
10764; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10765; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10766; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
10767; GFX11-WGP-NEXT:    s_endpgm
10768;
10769; GFX11-CU-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10770; GFX11-CU:       ; %bb.0: ; %entry
10771; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10772; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10773; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
10774; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
10775; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10776; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10777; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10778; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10779; GFX11-CU-NEXT:    s_endpgm
10780    i32 addrspace(3)* %out, i32 %in, i32 %old) {
10781entry:
10782  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
10783  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
10784  %val0 = extractvalue { i32, i1 } %val, 0
10785  store i32 %val0, i32 addrspace(3)* %out, align 4
10786  ret void
10787}
10788
10789define amdgpu_kernel void @local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
10790; GFX6-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10791; GFX6:       ; %bb.0: ; %entry
10792; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
10793; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
10794; GFX6-NEXT:    s_mov_b32 m0, -1
10795; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10796; GFX6-NEXT:    v_mov_b32_e32 v0, s2
10797; GFX6-NEXT:    v_mov_b32_e32 v1, s1
10798; GFX6-NEXT:    v_mov_b32_e32 v2, s0
10799; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10800; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10801; GFX6-NEXT:    ds_write_b32 v0, v1
10802; GFX6-NEXT:    s_endpgm
10803;
10804; GFX7-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10805; GFX7:       ; %bb.0: ; %entry
10806; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10807; GFX7-NEXT:    s_mov_b32 m0, -1
10808; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10809; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10810; GFX7-NEXT:    v_mov_b32_e32 v1, s2
10811; GFX7-NEXT:    v_mov_b32_e32 v2, s1
10812; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10813; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10814; GFX7-NEXT:    ds_write_b32 v0, v1
10815; GFX7-NEXT:    s_endpgm
10816;
10817; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10818; GFX10-WGP:       ; %bb.0: ; %entry
10819; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10820; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10821; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10822; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
10823; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
10824; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10825; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10826; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
10827; GFX10-WGP-NEXT:    s_endpgm
10828;
10829; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10830; GFX10-CU:       ; %bb.0: ; %entry
10831; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10832; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10833; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10834; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10835; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10836; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10837; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10838; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10839; GFX10-CU-NEXT:    s_endpgm
10840;
10841; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10842; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10843; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10844; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
10845; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10846; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
10847; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
10848; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
10849; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10850; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10851; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
10852; SKIP-CACHE-INV-NEXT:    s_endpgm
10853;
10854; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10855; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10856; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10857; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10858; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10859; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10860; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10861; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10862; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10863; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10864; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10865;
10866; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10867; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10868; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10869; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10870; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10871; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10872; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10873; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10874; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10875; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10876; GFX90A-TGSPLIT-NEXT:    s_endpgm
10877;
10878; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10879; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10880; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10881; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10882; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10883; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10884; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10885; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10886; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10887; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10888; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10889;
10890; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10891; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10892; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10893; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10894; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10895; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10896; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10897; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10898; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10899; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10900; GFX940-TGSPLIT-NEXT:    s_endpgm
10901;
10902; GFX11-WGP-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10903; GFX11-WGP:       ; %bb.0: ; %entry
10904; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10905; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10906; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
10907; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
10908; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
10909; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10910; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10911; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
10912; GFX11-WGP-NEXT:    s_endpgm
10913;
10914; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10915; GFX11-CU:       ; %bb.0: ; %entry
10916; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10917; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10918; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
10919; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
10920; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10921; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10922; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10923; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10924; GFX11-CU-NEXT:    s_endpgm
10925    i32 addrspace(3)* %out, i32 %in, i32 %old) {
10926entry:
10927  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
10928  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
10929  %val0 = extractvalue { i32, i1 } %val, 0
10930  store i32 %val0, i32 addrspace(3)* %out, align 4
10931  ret void
10932}
10933
10934define amdgpu_kernel void @local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
10935; GFX6-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
10936; GFX6:       ; %bb.0: ; %entry
10937; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
10938; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
10939; GFX6-NEXT:    s_mov_b32 m0, -1
10940; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10941; GFX6-NEXT:    v_mov_b32_e32 v0, s2
10942; GFX6-NEXT:    v_mov_b32_e32 v1, s1
10943; GFX6-NEXT:    v_mov_b32_e32 v2, s0
10944; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10945; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10946; GFX6-NEXT:    ds_write_b32 v0, v1
10947; GFX6-NEXT:    s_endpgm
10948;
10949; GFX7-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
10950; GFX7:       ; %bb.0: ; %entry
10951; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10952; GFX7-NEXT:    s_mov_b32 m0, -1
10953; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10954; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10955; GFX7-NEXT:    v_mov_b32_e32 v1, s2
10956; GFX7-NEXT:    v_mov_b32_e32 v2, s1
10957; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10958; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10959; GFX7-NEXT:    ds_write_b32 v0, v1
10960; GFX7-NEXT:    s_endpgm
10961;
10962; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
10963; GFX10-WGP:       ; %bb.0: ; %entry
10964; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10965; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10966; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10967; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
10968; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
10969; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10970; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10971; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
10972; GFX10-WGP-NEXT:    s_endpgm
10973;
10974; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
10975; GFX10-CU:       ; %bb.0: ; %entry
10976; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10977; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10978; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10979; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10980; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10981; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10982; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10983; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10984; GFX10-CU-NEXT:    s_endpgm
10985;
10986; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
10987; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10988; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10989; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
10990; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10991; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
10992; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
10993; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
10994; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10995; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10996; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
10997; SKIP-CACHE-INV-NEXT:    s_endpgm
10998;
10999; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11000; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11001; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11002; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11003; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11004; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11005; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11006; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11007; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11008; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11009; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11010;
11011; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11012; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11013; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11014; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11015; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11016; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11017; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11018; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11019; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11020; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11021; GFX90A-TGSPLIT-NEXT:    s_endpgm
11022;
11023; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11024; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11025; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11026; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11027; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11028; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11029; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11030; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11031; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11032; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11033; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11034;
11035; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11036; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11037; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11038; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11039; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11040; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11041; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11042; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11043; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11044; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11045; GFX940-TGSPLIT-NEXT:    s_endpgm
11046;
11047; GFX11-WGP-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11048; GFX11-WGP:       ; %bb.0: ; %entry
11049; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11050; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11051; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
11052; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
11053; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
11054; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11055; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11056; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
11057; GFX11-WGP-NEXT:    s_endpgm
11058;
11059; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11060; GFX11-CU:       ; %bb.0: ; %entry
11061; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11062; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11063; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
11064; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
11065; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
11066; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11067; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11068; GFX11-CU-NEXT:    ds_store_b32 v0, v1
11069; GFX11-CU-NEXT:    s_endpgm
11070    i32 addrspace(3)* %out, i32 %in, i32 %old) {
11071entry:
11072  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
11073  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
11074  %val0 = extractvalue { i32, i1 } %val, 0
11075  store i32 %val0, i32 addrspace(3)* %out, align 4
11076  ret void
11077}
11078
11079define amdgpu_kernel void @local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg(
11080; GFX6-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
11081; GFX6:       ; %bb.0: ; %entry
11082; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
11083; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
11084; GFX6-NEXT:    s_mov_b32 m0, -1
11085; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11086; GFX6-NEXT:    v_mov_b32_e32 v0, s2
11087; GFX6-NEXT:    v_mov_b32_e32 v1, s1
11088; GFX6-NEXT:    v_mov_b32_e32 v2, s0
11089; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11090; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11091; GFX6-NEXT:    ds_write_b32 v0, v1
11092; GFX6-NEXT:    s_endpgm
11093;
11094; GFX7-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
11095; GFX7:       ; %bb.0: ; %entry
11096; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11097; GFX7-NEXT:    s_mov_b32 m0, -1
11098; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11099; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11100; GFX7-NEXT:    v_mov_b32_e32 v1, s2
11101; GFX7-NEXT:    v_mov_b32_e32 v2, s1
11102; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11103; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11104; GFX7-NEXT:    ds_write_b32 v0, v1
11105; GFX7-NEXT:    s_endpgm
11106;
11107; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
11108; GFX10-WGP:       ; %bb.0: ; %entry
11109; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11110; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11111; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11112; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
11113; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
11114; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11115; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11116; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
11117; GFX10-WGP-NEXT:    s_endpgm
11118;
11119; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
11120; GFX10-CU:       ; %bb.0: ; %entry
11121; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11122; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11123; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11124; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
11125; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
11126; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11127; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11128; GFX10-CU-NEXT:    ds_write_b32 v0, v1
11129; GFX10-CU-NEXT:    s_endpgm
11130;
11131; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
11132; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11133; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11134; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
11135; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11136; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
11137; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
11138; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
11139; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11140; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11141; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
11142; SKIP-CACHE-INV-NEXT:    s_endpgm
11143;
11144; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
11145; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11146; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11147; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11148; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11149; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11150; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11151; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11152; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11153; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11154; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11155;
11156; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
11157; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11158; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11159; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11160; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11161; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11162; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11163; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11164; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11165; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11166; GFX90A-TGSPLIT-NEXT:    s_endpgm
11167;
11168; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
11169; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11170; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11171; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11172; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11173; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11174; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11175; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11176; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11177; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11178; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11179;
11180; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
11181; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11182; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11183; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11184; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11185; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11186; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11187; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11188; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11189; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11190; GFX940-TGSPLIT-NEXT:    s_endpgm
11191;
11192; GFX11-WGP-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
11193; GFX11-WGP:       ; %bb.0: ; %entry
11194; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11195; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11196; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
11197; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
11198; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
11199; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11200; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11201; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
11202; GFX11-WGP-NEXT:    s_endpgm
11203;
11204; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
11205; GFX11-CU:       ; %bb.0: ; %entry
11206; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11207; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11208; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
11209; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
11210; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
11211; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11212; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11213; GFX11-CU-NEXT:    ds_store_b32 v0, v1
11214; GFX11-CU-NEXT:    s_endpgm
11215    i32 addrspace(3)* %out, i32 %in, i32 %old) {
11216entry:
11217  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
11218  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic seq_cst
11219  %val0 = extractvalue { i32, i1 } %val, 0
11220  store i32 %val0, i32 addrspace(3)* %out, align 4
11221  ret void
11222}
11223
11224define amdgpu_kernel void @local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg(
11225; GFX6-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11226; GFX6:       ; %bb.0: ; %entry
11227; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
11228; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
11229; GFX6-NEXT:    s_mov_b32 m0, -1
11230; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11231; GFX6-NEXT:    v_mov_b32_e32 v0, s2
11232; GFX6-NEXT:    v_mov_b32_e32 v1, s1
11233; GFX6-NEXT:    v_mov_b32_e32 v2, s0
11234; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11235; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11236; GFX6-NEXT:    ds_write_b32 v0, v1
11237; GFX6-NEXT:    s_endpgm
11238;
11239; GFX7-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11240; GFX7:       ; %bb.0: ; %entry
11241; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11242; GFX7-NEXT:    s_mov_b32 m0, -1
11243; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11244; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11245; GFX7-NEXT:    v_mov_b32_e32 v1, s2
11246; GFX7-NEXT:    v_mov_b32_e32 v2, s1
11247; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11248; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11249; GFX7-NEXT:    ds_write_b32 v0, v1
11250; GFX7-NEXT:    s_endpgm
11251;
11252; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11253; GFX10-WGP:       ; %bb.0: ; %entry
11254; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11255; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11256; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11257; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
11258; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
11259; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11260; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11261; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
11262; GFX10-WGP-NEXT:    s_endpgm
11263;
11264; GFX10-CU-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11265; GFX10-CU:       ; %bb.0: ; %entry
11266; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11267; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11268; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11269; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
11270; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
11271; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11272; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11273; GFX10-CU-NEXT:    ds_write_b32 v0, v1
11274; GFX10-CU-NEXT:    s_endpgm
11275;
11276; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11277; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11278; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11279; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
11280; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11281; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
11282; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
11283; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
11284; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11285; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11286; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
11287; SKIP-CACHE-INV-NEXT:    s_endpgm
11288;
11289; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11290; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11291; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11292; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11293; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11294; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11295; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11296; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11297; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11298; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11299; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11300;
11301; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11302; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11303; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11304; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11305; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11306; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11307; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11308; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11309; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11310; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11311; GFX90A-TGSPLIT-NEXT:    s_endpgm
11312;
11313; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11314; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11315; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11316; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11317; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11318; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11319; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11320; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11321; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11322; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11323; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11324;
11325; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11326; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11327; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11328; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11329; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11330; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11331; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11332; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11333; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11334; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11335; GFX940-TGSPLIT-NEXT:    s_endpgm
11336;
11337; GFX11-WGP-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11338; GFX11-WGP:       ; %bb.0: ; %entry
11339; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11340; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11341; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
11342; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
11343; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
11344; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11345; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11346; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
11347; GFX11-WGP-NEXT:    s_endpgm
11348;
11349; GFX11-CU-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11350; GFX11-CU:       ; %bb.0: ; %entry
11351; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11352; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11353; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
11354; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
11355; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
11356; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11357; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11358; GFX11-CU-NEXT:    ds_store_b32 v0, v1
11359; GFX11-CU-NEXT:    s_endpgm
11360    i32 addrspace(3)* %out, i32 %in, i32 %old) {
11361entry:
11362  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
11363  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire seq_cst
11364  %val0 = extractvalue { i32, i1 } %val, 0
11365  store i32 %val0, i32 addrspace(3)* %out, align 4
11366  ret void
11367}
11368
11369define amdgpu_kernel void @local_singlethread_one_as_release_seq_cst_ret_cmpxchg(
11370; GFX6-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11371; GFX6:       ; %bb.0: ; %entry
11372; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
11373; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
11374; GFX6-NEXT:    s_mov_b32 m0, -1
11375; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11376; GFX6-NEXT:    v_mov_b32_e32 v0, s2
11377; GFX6-NEXT:    v_mov_b32_e32 v1, s1
11378; GFX6-NEXT:    v_mov_b32_e32 v2, s0
11379; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11380; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11381; GFX6-NEXT:    ds_write_b32 v0, v1
11382; GFX6-NEXT:    s_endpgm
11383;
11384; GFX7-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11385; GFX7:       ; %bb.0: ; %entry
11386; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11387; GFX7-NEXT:    s_mov_b32 m0, -1
11388; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11389; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11390; GFX7-NEXT:    v_mov_b32_e32 v1, s2
11391; GFX7-NEXT:    v_mov_b32_e32 v2, s1
11392; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11393; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11394; GFX7-NEXT:    ds_write_b32 v0, v1
11395; GFX7-NEXT:    s_endpgm
11396;
11397; GFX10-WGP-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11398; GFX10-WGP:       ; %bb.0: ; %entry
11399; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11400; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11401; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11402; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
11403; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
11404; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11405; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11406; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
11407; GFX10-WGP-NEXT:    s_endpgm
11408;
11409; GFX10-CU-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11410; GFX10-CU:       ; %bb.0: ; %entry
11411; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11412; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11413; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11414; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
11415; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
11416; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11417; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11418; GFX10-CU-NEXT:    ds_write_b32 v0, v1
11419; GFX10-CU-NEXT:    s_endpgm
11420;
11421; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11422; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11423; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11424; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
11425; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11426; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
11427; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
11428; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
11429; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11430; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11431; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
11432; SKIP-CACHE-INV-NEXT:    s_endpgm
11433;
11434; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11435; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11436; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11437; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11438; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11439; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11440; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11441; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11442; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11443; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11444; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11445;
11446; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11447; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11448; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11449; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11450; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11451; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11452; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11453; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11454; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11455; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11456; GFX90A-TGSPLIT-NEXT:    s_endpgm
11457;
11458; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11459; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11460; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11461; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11462; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11463; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11464; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11465; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11466; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11467; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11468; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11469;
11470; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11471; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11472; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11473; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11474; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11475; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11476; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11477; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11478; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11479; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11480; GFX940-TGSPLIT-NEXT:    s_endpgm
11481;
11482; GFX11-WGP-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11483; GFX11-WGP:       ; %bb.0: ; %entry
11484; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11485; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11486; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
11487; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
11488; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
11489; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11490; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11491; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
11492; GFX11-WGP-NEXT:    s_endpgm
11493;
11494; GFX11-CU-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11495; GFX11-CU:       ; %bb.0: ; %entry
11496; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11497; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11498; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
11499; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
11500; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
11501; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11502; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11503; GFX11-CU-NEXT:    ds_store_b32 v0, v1
11504; GFX11-CU-NEXT:    s_endpgm
11505    i32 addrspace(3)* %out, i32 %in, i32 %old) {
11506entry:
11507  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
11508  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release seq_cst
11509  %val0 = extractvalue { i32, i1 } %val, 0
11510  store i32 %val0, i32 addrspace(3)* %out, align 4
11511  ret void
11512}
11513
11514define amdgpu_kernel void @local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg(
11515; GFX6-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11516; GFX6:       ; %bb.0: ; %entry
11517; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
11518; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
11519; GFX6-NEXT:    s_mov_b32 m0, -1
11520; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11521; GFX6-NEXT:    v_mov_b32_e32 v0, s2
11522; GFX6-NEXT:    v_mov_b32_e32 v1, s1
11523; GFX6-NEXT:    v_mov_b32_e32 v2, s0
11524; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11525; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11526; GFX6-NEXT:    ds_write_b32 v0, v1
11527; GFX6-NEXT:    s_endpgm
11528;
11529; GFX7-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11530; GFX7:       ; %bb.0: ; %entry
11531; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11532; GFX7-NEXT:    s_mov_b32 m0, -1
11533; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11534; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11535; GFX7-NEXT:    v_mov_b32_e32 v1, s2
11536; GFX7-NEXT:    v_mov_b32_e32 v2, s1
11537; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11538; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11539; GFX7-NEXT:    ds_write_b32 v0, v1
11540; GFX7-NEXT:    s_endpgm
11541;
11542; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11543; GFX10-WGP:       ; %bb.0: ; %entry
11544; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11545; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11546; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11547; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
11548; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
11549; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11550; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11551; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
11552; GFX10-WGP-NEXT:    s_endpgm
11553;
11554; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11555; GFX10-CU:       ; %bb.0: ; %entry
11556; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11557; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11558; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11559; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
11560; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
11561; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11562; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11563; GFX10-CU-NEXT:    ds_write_b32 v0, v1
11564; GFX10-CU-NEXT:    s_endpgm
11565;
11566; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11567; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11568; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11569; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
11570; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11571; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
11572; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
11573; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
11574; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11575; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11576; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
11577; SKIP-CACHE-INV-NEXT:    s_endpgm
11578;
11579; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11580; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11581; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11582; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11583; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11584; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11585; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11586; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11587; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11588; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11589; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11590;
11591; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11592; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11593; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11594; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11595; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11596; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11597; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11598; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11599; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11600; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11601; GFX90A-TGSPLIT-NEXT:    s_endpgm
11602;
11603; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11604; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11605; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11606; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11607; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11608; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11609; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11610; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11611; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11612; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11613; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11614;
11615; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11616; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11617; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11618; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11619; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11620; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11621; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11622; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11623; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11624; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11625; GFX940-TGSPLIT-NEXT:    s_endpgm
11626;
11627; GFX11-WGP-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11628; GFX11-WGP:       ; %bb.0: ; %entry
11629; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11630; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11631; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
11632; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
11633; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
11634; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11635; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11636; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
11637; GFX11-WGP-NEXT:    s_endpgm
11638;
11639; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11640; GFX11-CU:       ; %bb.0: ; %entry
11641; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11642; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11643; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
11644; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
11645; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
11646; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11647; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11648; GFX11-CU-NEXT:    ds_store_b32 v0, v1
11649; GFX11-CU-NEXT:    s_endpgm
11650    i32 addrspace(3)* %out, i32 %in, i32 %old) {
11651entry:
11652  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
11653  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel seq_cst
11654  %val0 = extractvalue { i32, i1 } %val, 0
11655  store i32 %val0, i32 addrspace(3)* %out, align 4
11656  ret void
11657}
11658
11659define amdgpu_kernel void @local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
11660; GFX6-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11661; GFX6:       ; %bb.0: ; %entry
11662; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
11663; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
11664; GFX6-NEXT:    s_mov_b32 m0, -1
11665; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11666; GFX6-NEXT:    v_mov_b32_e32 v0, s2
11667; GFX6-NEXT:    v_mov_b32_e32 v1, s1
11668; GFX6-NEXT:    v_mov_b32_e32 v2, s0
11669; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11670; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11671; GFX6-NEXT:    ds_write_b32 v0, v1
11672; GFX6-NEXT:    s_endpgm
11673;
11674; GFX7-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11675; GFX7:       ; %bb.0: ; %entry
11676; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11677; GFX7-NEXT:    s_mov_b32 m0, -1
11678; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11679; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11680; GFX7-NEXT:    v_mov_b32_e32 v1, s2
11681; GFX7-NEXT:    v_mov_b32_e32 v2, s1
11682; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11683; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11684; GFX7-NEXT:    ds_write_b32 v0, v1
11685; GFX7-NEXT:    s_endpgm
11686;
11687; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11688; GFX10-WGP:       ; %bb.0: ; %entry
11689; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11690; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11691; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11692; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
11693; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
11694; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11695; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11696; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
11697; GFX10-WGP-NEXT:    s_endpgm
11698;
11699; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11700; GFX10-CU:       ; %bb.0: ; %entry
11701; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11702; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11703; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11704; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
11705; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
11706; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11707; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11708; GFX10-CU-NEXT:    ds_write_b32 v0, v1
11709; GFX10-CU-NEXT:    s_endpgm
11710;
11711; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11712; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11713; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11714; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
11715; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11716; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
11717; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
11718; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
11719; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11720; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11721; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
11722; SKIP-CACHE-INV-NEXT:    s_endpgm
11723;
11724; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11725; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11726; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11727; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11728; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11729; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11730; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11731; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11732; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11733; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11734; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11735;
11736; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11737; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11738; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11739; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11740; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11741; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11742; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11743; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11744; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11745; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11746; GFX90A-TGSPLIT-NEXT:    s_endpgm
11747;
11748; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11749; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11750; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11751; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11752; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11753; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11754; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11755; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11756; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11757; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11758; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11759;
11760; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11761; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11762; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11763; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11764; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11765; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11766; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11767; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11768; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11769; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11770; GFX940-TGSPLIT-NEXT:    s_endpgm
11771;
11772; GFX11-WGP-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11773; GFX11-WGP:       ; %bb.0: ; %entry
11774; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11775; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11776; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
11777; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
11778; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
11779; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11780; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11781; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
11782; GFX11-WGP-NEXT:    s_endpgm
11783;
11784; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11785; GFX11-CU:       ; %bb.0: ; %entry
11786; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11787; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11788; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
11789; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
11790; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
11791; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11792; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11793; GFX11-CU-NEXT:    ds_store_b32 v0, v1
11794; GFX11-CU-NEXT:    s_endpgm
11795    i32 addrspace(3)* %out, i32 %in, i32 %old) {
11796entry:
11797  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
11798  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
11799  %val0 = extractvalue { i32, i1 } %val, 0
11800  store i32 %val0, i32 addrspace(3)* %out, align 4
11801  ret void
11802}
11803
11804