1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
5; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
6; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
8; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
9; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
10; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
11; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-WGP %s
12; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-CU %s
13
14define amdgpu_kernel void @local_singlethread_unordered_load(
15; GFX6-LABEL: local_singlethread_unordered_load:
16; GFX6:       ; %bb.0: ; %entry
17; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
18; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
19; GFX6-NEXT:    s_mov_b32 m0, -1
20; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
21; GFX6-NEXT:    v_mov_b32_e32 v0, s0
22; GFX6-NEXT:    ds_read_b32 v0, v0
23; GFX6-NEXT:    v_mov_b32_e32 v1, s1
24; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
25; GFX6-NEXT:    ds_write_b32 v1, v0
26; GFX6-NEXT:    s_endpgm
27;
28; GFX7-LABEL: local_singlethread_unordered_load:
29; GFX7:       ; %bb.0: ; %entry
30; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
31; GFX7-NEXT:    s_mov_b32 m0, -1
32; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
33; GFX7-NEXT:    v_mov_b32_e32 v0, s0
34; GFX7-NEXT:    ds_read_b32 v0, v0
35; GFX7-NEXT:    v_mov_b32_e32 v1, s1
36; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
37; GFX7-NEXT:    ds_write_b32 v1, v0
38; GFX7-NEXT:    s_endpgm
39;
40; GFX10-WGP-LABEL: local_singlethread_unordered_load:
41; GFX10-WGP:       ; %bb.0: ; %entry
42; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
43; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
44; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
45; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
46; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
47; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
48; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
49; GFX10-WGP-NEXT:    s_endpgm
50;
51; GFX10-CU-LABEL: local_singlethread_unordered_load:
52; GFX10-CU:       ; %bb.0: ; %entry
53; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
54; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
55; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
56; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
57; GFX10-CU-NEXT:    ds_read_b32 v0, v0
58; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
59; GFX10-CU-NEXT:    ds_write_b32 v1, v0
60; GFX10-CU-NEXT:    s_endpgm
61;
62; SKIP-CACHE-INV-LABEL: local_singlethread_unordered_load:
63; SKIP-CACHE-INV:       ; %bb.0: ; %entry
64; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
65; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
66; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
67; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
68; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
69; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
70; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
71; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
72; SKIP-CACHE-INV-NEXT:    s_endpgm
73;
74; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_unordered_load:
75; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
76; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
77; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
78; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
79; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
80; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
81; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
82; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
83; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
84;
85; GFX90A-TGSPLIT-LABEL: local_singlethread_unordered_load:
86; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
87; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
88; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
89; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
90; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
91; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
92; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
93; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
94; GFX90A-TGSPLIT-NEXT:    s_endpgm
95;
96; GFX940-NOTTGSPLIT-LABEL: local_singlethread_unordered_load:
97; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
98; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
99; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
100; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
101; GFX940-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
102; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
103; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
104; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
105; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
106;
107; GFX940-TGSPLIT-LABEL: local_singlethread_unordered_load:
108; GFX940-TGSPLIT:       ; %bb.0: ; %entry
109; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
110; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
111; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
112; GFX940-TGSPLIT-NEXT:    ds_read_b32 v0, v0
113; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
114; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
115; GFX940-TGSPLIT-NEXT:    ds_write_b32 v1, v0
116; GFX940-TGSPLIT-NEXT:    s_endpgm
117;
118; GFX11-WGP-LABEL: local_singlethread_unordered_load:
119; GFX11-WGP:       ; %bb.0: ; %entry
120; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
121; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
122; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
123; GFX11-WGP-NEXT:    ds_load_b32 v0, v0
124; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
125; GFX11-WGP-NEXT:    ds_store_b32 v1, v0
126; GFX11-WGP-NEXT:    s_endpgm
127;
128; GFX11-CU-LABEL: local_singlethread_unordered_load:
129; GFX11-CU:       ; %bb.0: ; %entry
130; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
131; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
132; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
133; GFX11-CU-NEXT:    ds_load_b32 v0, v0
134; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
135; GFX11-CU-NEXT:    ds_store_b32 v1, v0
136; GFX11-CU-NEXT:    s_endpgm
137    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
138entry:
139  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread") unordered, align 4
140  store i32 %val, i32 addrspace(3)* %out
141  ret void
142}
143
144define amdgpu_kernel void @local_singlethread_monotonic_load(
145; GFX6-LABEL: local_singlethread_monotonic_load:
146; GFX6:       ; %bb.0: ; %entry
147; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
148; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
149; GFX6-NEXT:    s_mov_b32 m0, -1
150; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
151; GFX6-NEXT:    v_mov_b32_e32 v0, s0
152; GFX6-NEXT:    ds_read_b32 v0, v0
153; GFX6-NEXT:    v_mov_b32_e32 v1, s1
154; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
155; GFX6-NEXT:    ds_write_b32 v1, v0
156; GFX6-NEXT:    s_endpgm
157;
158; GFX7-LABEL: local_singlethread_monotonic_load:
159; GFX7:       ; %bb.0: ; %entry
160; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
161; GFX7-NEXT:    s_mov_b32 m0, -1
162; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
163; GFX7-NEXT:    v_mov_b32_e32 v0, s0
164; GFX7-NEXT:    ds_read_b32 v0, v0
165; GFX7-NEXT:    v_mov_b32_e32 v1, s1
166; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
167; GFX7-NEXT:    ds_write_b32 v1, v0
168; GFX7-NEXT:    s_endpgm
169;
170; GFX10-WGP-LABEL: local_singlethread_monotonic_load:
171; GFX10-WGP:       ; %bb.0: ; %entry
172; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
173; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
174; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
175; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
176; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
177; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
178; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
179; GFX10-WGP-NEXT:    s_endpgm
180;
181; GFX10-CU-LABEL: local_singlethread_monotonic_load:
182; GFX10-CU:       ; %bb.0: ; %entry
183; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
184; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
185; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
186; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
187; GFX10-CU-NEXT:    ds_read_b32 v0, v0
188; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
189; GFX10-CU-NEXT:    ds_write_b32 v1, v0
190; GFX10-CU-NEXT:    s_endpgm
191;
192; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_load:
193; SKIP-CACHE-INV:       ; %bb.0: ; %entry
194; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
195; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
196; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
197; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
198; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
199; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
200; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
201; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
202; SKIP-CACHE-INV-NEXT:    s_endpgm
203;
204; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_load:
205; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
206; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
207; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
208; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
209; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
210; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
211; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
212; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
213; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
214;
215; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_load:
216; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
217; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
218; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
219; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
220; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
221; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
222; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
223; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
224; GFX90A-TGSPLIT-NEXT:    s_endpgm
225;
226; GFX940-NOTTGSPLIT-LABEL: local_singlethread_monotonic_load:
227; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
228; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
229; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
230; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
231; GFX940-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
232; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
233; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
234; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
235; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
236;
237; GFX940-TGSPLIT-LABEL: local_singlethread_monotonic_load:
238; GFX940-TGSPLIT:       ; %bb.0: ; %entry
239; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
240; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
241; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
242; GFX940-TGSPLIT-NEXT:    ds_read_b32 v0, v0
243; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
244; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
245; GFX940-TGSPLIT-NEXT:    ds_write_b32 v1, v0
246; GFX940-TGSPLIT-NEXT:    s_endpgm
247;
248; GFX11-WGP-LABEL: local_singlethread_monotonic_load:
249; GFX11-WGP:       ; %bb.0: ; %entry
250; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
251; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
252; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
253; GFX11-WGP-NEXT:    ds_load_b32 v0, v0
254; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
255; GFX11-WGP-NEXT:    ds_store_b32 v1, v0
256; GFX11-WGP-NEXT:    s_endpgm
257;
258; GFX11-CU-LABEL: local_singlethread_monotonic_load:
259; GFX11-CU:       ; %bb.0: ; %entry
260; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
261; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
262; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
263; GFX11-CU-NEXT:    ds_load_b32 v0, v0
264; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
265; GFX11-CU-NEXT:    ds_store_b32 v1, v0
266; GFX11-CU-NEXT:    s_endpgm
267    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
268entry:
269  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread") monotonic, align 4
270  store i32 %val, i32 addrspace(3)* %out
271  ret void
272}
273
274define amdgpu_kernel void @local_singlethread_acquire_load(
275; GFX6-LABEL: local_singlethread_acquire_load:
276; GFX6:       ; %bb.0: ; %entry
277; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
278; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
279; GFX6-NEXT:    s_mov_b32 m0, -1
280; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
281; GFX6-NEXT:    v_mov_b32_e32 v0, s0
282; GFX6-NEXT:    ds_read_b32 v0, v0
283; GFX6-NEXT:    v_mov_b32_e32 v1, s1
284; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
285; GFX6-NEXT:    ds_write_b32 v1, v0
286; GFX6-NEXT:    s_endpgm
287;
288; GFX7-LABEL: local_singlethread_acquire_load:
289; GFX7:       ; %bb.0: ; %entry
290; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
291; GFX7-NEXT:    s_mov_b32 m0, -1
292; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
293; GFX7-NEXT:    v_mov_b32_e32 v0, s0
294; GFX7-NEXT:    ds_read_b32 v0, v0
295; GFX7-NEXT:    v_mov_b32_e32 v1, s1
296; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
297; GFX7-NEXT:    ds_write_b32 v1, v0
298; GFX7-NEXT:    s_endpgm
299;
300; GFX10-WGP-LABEL: local_singlethread_acquire_load:
301; GFX10-WGP:       ; %bb.0: ; %entry
302; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
303; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
304; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
305; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
306; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
307; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
308; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
309; GFX10-WGP-NEXT:    s_endpgm
310;
311; GFX10-CU-LABEL: local_singlethread_acquire_load:
312; GFX10-CU:       ; %bb.0: ; %entry
313; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
314; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
315; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
316; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
317; GFX10-CU-NEXT:    ds_read_b32 v0, v0
318; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
319; GFX10-CU-NEXT:    ds_write_b32 v1, v0
320; GFX10-CU-NEXT:    s_endpgm
321;
322; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_load:
323; SKIP-CACHE-INV:       ; %bb.0: ; %entry
324; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
325; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
326; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
327; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
328; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
329; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
330; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
331; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
332; SKIP-CACHE-INV-NEXT:    s_endpgm
333;
334; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_load:
335; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
336; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
337; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
338; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
339; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
340; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
341; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
342; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
343; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
344;
345; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_load:
346; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
347; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
348; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
349; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
350; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
351; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
352; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
353; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
354; GFX90A-TGSPLIT-NEXT:    s_endpgm
355;
356; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acquire_load:
357; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
358; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
359; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
360; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
361; GFX940-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
362; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
363; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
364; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
365; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
366;
367; GFX940-TGSPLIT-LABEL: local_singlethread_acquire_load:
368; GFX940-TGSPLIT:       ; %bb.0: ; %entry
369; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
370; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
371; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
372; GFX940-TGSPLIT-NEXT:    ds_read_b32 v0, v0
373; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
374; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
375; GFX940-TGSPLIT-NEXT:    ds_write_b32 v1, v0
376; GFX940-TGSPLIT-NEXT:    s_endpgm
377;
378; GFX11-WGP-LABEL: local_singlethread_acquire_load:
379; GFX11-WGP:       ; %bb.0: ; %entry
380; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
381; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
382; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
383; GFX11-WGP-NEXT:    ds_load_b32 v0, v0
384; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
385; GFX11-WGP-NEXT:    ds_store_b32 v1, v0
386; GFX11-WGP-NEXT:    s_endpgm
387;
388; GFX11-CU-LABEL: local_singlethread_acquire_load:
389; GFX11-CU:       ; %bb.0: ; %entry
390; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
391; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
392; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
393; GFX11-CU-NEXT:    ds_load_b32 v0, v0
394; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
395; GFX11-CU-NEXT:    ds_store_b32 v1, v0
396; GFX11-CU-NEXT:    s_endpgm
397    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
398entry:
399  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread") acquire, align 4
400  store i32 %val, i32 addrspace(3)* %out
401  ret void
402}
403
404define amdgpu_kernel void @local_singlethread_seq_cst_load(
405; GFX6-LABEL: local_singlethread_seq_cst_load:
406; GFX6:       ; %bb.0: ; %entry
407; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
408; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
409; GFX6-NEXT:    s_mov_b32 m0, -1
410; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
411; GFX6-NEXT:    v_mov_b32_e32 v0, s0
412; GFX6-NEXT:    ds_read_b32 v0, v0
413; GFX6-NEXT:    v_mov_b32_e32 v1, s1
414; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
415; GFX6-NEXT:    ds_write_b32 v1, v0
416; GFX6-NEXT:    s_endpgm
417;
418; GFX7-LABEL: local_singlethread_seq_cst_load:
419; GFX7:       ; %bb.0: ; %entry
420; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
421; GFX7-NEXT:    s_mov_b32 m0, -1
422; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
423; GFX7-NEXT:    v_mov_b32_e32 v0, s0
424; GFX7-NEXT:    ds_read_b32 v0, v0
425; GFX7-NEXT:    v_mov_b32_e32 v1, s1
426; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
427; GFX7-NEXT:    ds_write_b32 v1, v0
428; GFX7-NEXT:    s_endpgm
429;
430; GFX10-WGP-LABEL: local_singlethread_seq_cst_load:
431; GFX10-WGP:       ; %bb.0: ; %entry
432; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
433; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
434; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
435; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
436; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
437; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
438; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
439; GFX10-WGP-NEXT:    s_endpgm
440;
441; GFX10-CU-LABEL: local_singlethread_seq_cst_load:
442; GFX10-CU:       ; %bb.0: ; %entry
443; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
444; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
445; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
446; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
447; GFX10-CU-NEXT:    ds_read_b32 v0, v0
448; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
449; GFX10-CU-NEXT:    ds_write_b32 v1, v0
450; GFX10-CU-NEXT:    s_endpgm
451;
452; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_load:
453; SKIP-CACHE-INV:       ; %bb.0: ; %entry
454; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
455; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
456; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
457; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
458; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
459; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
460; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
461; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
462; SKIP-CACHE-INV-NEXT:    s_endpgm
463;
464; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_load:
465; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
466; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
467; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
468; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
469; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
470; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
471; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
472; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
473; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
474;
475; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_load:
476; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
477; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
478; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
479; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
480; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
481; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
482; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
483; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
484; GFX90A-TGSPLIT-NEXT:    s_endpgm
485;
486; GFX940-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_load:
487; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
488; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
489; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
490; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
491; GFX940-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
492; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
493; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
494; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
495; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
496;
497; GFX940-TGSPLIT-LABEL: local_singlethread_seq_cst_load:
498; GFX940-TGSPLIT:       ; %bb.0: ; %entry
499; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
500; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
501; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
502; GFX940-TGSPLIT-NEXT:    ds_read_b32 v0, v0
503; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
504; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
505; GFX940-TGSPLIT-NEXT:    ds_write_b32 v1, v0
506; GFX940-TGSPLIT-NEXT:    s_endpgm
507;
508; GFX11-WGP-LABEL: local_singlethread_seq_cst_load:
509; GFX11-WGP:       ; %bb.0: ; %entry
510; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
511; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
512; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
513; GFX11-WGP-NEXT:    ds_load_b32 v0, v0
514; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
515; GFX11-WGP-NEXT:    ds_store_b32 v1, v0
516; GFX11-WGP-NEXT:    s_endpgm
517;
518; GFX11-CU-LABEL: local_singlethread_seq_cst_load:
519; GFX11-CU:       ; %bb.0: ; %entry
520; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
521; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
522; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
523; GFX11-CU-NEXT:    ds_load_b32 v0, v0
524; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
525; GFX11-CU-NEXT:    ds_store_b32 v1, v0
526; GFX11-CU-NEXT:    s_endpgm
527    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
528entry:
529  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread") seq_cst, align 4
530  store i32 %val, i32 addrspace(3)* %out
531  ret void
532}
533
534define amdgpu_kernel void @local_singlethread_unordered_store(
535; GFX6-LABEL: local_singlethread_unordered_store:
536; GFX6:       ; %bb.0: ; %entry
537; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
538; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
539; GFX6-NEXT:    s_mov_b32 m0, -1
540; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
541; GFX6-NEXT:    v_mov_b32_e32 v0, s0
542; GFX6-NEXT:    v_mov_b32_e32 v1, s1
543; GFX6-NEXT:    ds_write_b32 v0, v1
544; GFX6-NEXT:    s_endpgm
545;
546; GFX7-LABEL: local_singlethread_unordered_store:
547; GFX7:       ; %bb.0: ; %entry
548; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
549; GFX7-NEXT:    s_mov_b32 m0, -1
550; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
551; GFX7-NEXT:    v_mov_b32_e32 v0, s1
552; GFX7-NEXT:    v_mov_b32_e32 v1, s0
553; GFX7-NEXT:    ds_write_b32 v0, v1
554; GFX7-NEXT:    s_endpgm
555;
556; GFX10-WGP-LABEL: local_singlethread_unordered_store:
557; GFX10-WGP:       ; %bb.0: ; %entry
558; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
559; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
560; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
561; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
562; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
563; GFX10-WGP-NEXT:    s_endpgm
564;
565; GFX10-CU-LABEL: local_singlethread_unordered_store:
566; GFX10-CU:       ; %bb.0: ; %entry
567; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
568; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
569; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
570; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
571; GFX10-CU-NEXT:    ds_write_b32 v0, v1
572; GFX10-CU-NEXT:    s_endpgm
573;
574; SKIP-CACHE-INV-LABEL: local_singlethread_unordered_store:
575; SKIP-CACHE-INV:       ; %bb.0: ; %entry
576; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
577; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
578; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
579; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
580; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
581; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
582; SKIP-CACHE-INV-NEXT:    s_endpgm
583;
584; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_unordered_store:
585; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
586; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
587; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
588; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
589; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
590; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
591; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
592;
593; GFX90A-TGSPLIT-LABEL: local_singlethread_unordered_store:
594; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
595; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
596; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
597; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
598; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
599; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
600; GFX90A-TGSPLIT-NEXT:    s_endpgm
601;
602; GFX940-NOTTGSPLIT-LABEL: local_singlethread_unordered_store:
603; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
604; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
605; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
606; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
607; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
608; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
609; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
610;
611; GFX940-TGSPLIT-LABEL: local_singlethread_unordered_store:
612; GFX940-TGSPLIT:       ; %bb.0: ; %entry
613; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
614; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
615; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
616; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
617; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
618; GFX940-TGSPLIT-NEXT:    s_endpgm
619;
620; GFX11-WGP-LABEL: local_singlethread_unordered_store:
621; GFX11-WGP:       ; %bb.0: ; %entry
622; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
623; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
624; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
625; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
626; GFX11-WGP-NEXT:    s_endpgm
627;
628; GFX11-CU-LABEL: local_singlethread_unordered_store:
629; GFX11-CU:       ; %bb.0: ; %entry
630; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
631; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
632; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
633; GFX11-CU-NEXT:    ds_store_b32 v0, v1
634; GFX11-CU-NEXT:    s_endpgm
635    i32 %in, i32 addrspace(3)* %out) {
636entry:
637  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread") unordered, align 4
638  ret void
639}
640
641define amdgpu_kernel void @local_singlethread_monotonic_store(
642; GFX6-LABEL: local_singlethread_monotonic_store:
643; GFX6:       ; %bb.0: ; %entry
644; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
645; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
646; GFX6-NEXT:    s_mov_b32 m0, -1
647; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
648; GFX6-NEXT:    v_mov_b32_e32 v0, s0
649; GFX6-NEXT:    v_mov_b32_e32 v1, s1
650; GFX6-NEXT:    ds_write_b32 v0, v1
651; GFX6-NEXT:    s_endpgm
652;
653; GFX7-LABEL: local_singlethread_monotonic_store:
654; GFX7:       ; %bb.0: ; %entry
655; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
656; GFX7-NEXT:    s_mov_b32 m0, -1
657; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
658; GFX7-NEXT:    v_mov_b32_e32 v0, s1
659; GFX7-NEXT:    v_mov_b32_e32 v1, s0
660; GFX7-NEXT:    ds_write_b32 v0, v1
661; GFX7-NEXT:    s_endpgm
662;
663; GFX10-WGP-LABEL: local_singlethread_monotonic_store:
664; GFX10-WGP:       ; %bb.0: ; %entry
665; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
666; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
667; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
668; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
669; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
670; GFX10-WGP-NEXT:    s_endpgm
671;
672; GFX10-CU-LABEL: local_singlethread_monotonic_store:
673; GFX10-CU:       ; %bb.0: ; %entry
674; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
675; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
676; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
677; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
678; GFX10-CU-NEXT:    ds_write_b32 v0, v1
679; GFX10-CU-NEXT:    s_endpgm
680;
681; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_store:
682; SKIP-CACHE-INV:       ; %bb.0: ; %entry
683; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
684; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
685; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
686; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
687; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
688; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
689; SKIP-CACHE-INV-NEXT:    s_endpgm
690;
691; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_store:
692; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
693; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
694; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
695; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
696; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
697; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
698; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
699;
700; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_store:
701; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
702; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
703; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
704; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
705; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
706; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
707; GFX90A-TGSPLIT-NEXT:    s_endpgm
708;
709; GFX940-NOTTGSPLIT-LABEL: local_singlethread_monotonic_store:
710; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
711; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
712; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
713; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
714; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
715; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
716; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
717;
718; GFX940-TGSPLIT-LABEL: local_singlethread_monotonic_store:
719; GFX940-TGSPLIT:       ; %bb.0: ; %entry
720; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
721; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
722; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
723; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
724; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
725; GFX940-TGSPLIT-NEXT:    s_endpgm
726;
727; GFX11-WGP-LABEL: local_singlethread_monotonic_store:
728; GFX11-WGP:       ; %bb.0: ; %entry
729; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
730; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
731; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
732; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
733; GFX11-WGP-NEXT:    s_endpgm
734;
735; GFX11-CU-LABEL: local_singlethread_monotonic_store:
736; GFX11-CU:       ; %bb.0: ; %entry
737; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
738; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
739; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
740; GFX11-CU-NEXT:    ds_store_b32 v0, v1
741; GFX11-CU-NEXT:    s_endpgm
742    i32 %in, i32 addrspace(3)* %out) {
743entry:
744  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread") monotonic, align 4
745  ret void
746}
747
748define amdgpu_kernel void @local_singlethread_release_store(
749; GFX6-LABEL: local_singlethread_release_store:
750; GFX6:       ; %bb.0: ; %entry
751; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
752; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
753; GFX6-NEXT:    s_mov_b32 m0, -1
754; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
755; GFX6-NEXT:    v_mov_b32_e32 v0, s0
756; GFX6-NEXT:    v_mov_b32_e32 v1, s1
757; GFX6-NEXT:    ds_write_b32 v0, v1
758; GFX6-NEXT:    s_endpgm
759;
760; GFX7-LABEL: local_singlethread_release_store:
761; GFX7:       ; %bb.0: ; %entry
762; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
763; GFX7-NEXT:    s_mov_b32 m0, -1
764; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
765; GFX7-NEXT:    v_mov_b32_e32 v0, s1
766; GFX7-NEXT:    v_mov_b32_e32 v1, s0
767; GFX7-NEXT:    ds_write_b32 v0, v1
768; GFX7-NEXT:    s_endpgm
769;
770; GFX10-WGP-LABEL: local_singlethread_release_store:
771; GFX10-WGP:       ; %bb.0: ; %entry
772; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
773; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
774; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
775; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
776; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
777; GFX10-WGP-NEXT:    s_endpgm
778;
779; GFX10-CU-LABEL: local_singlethread_release_store:
780; GFX10-CU:       ; %bb.0: ; %entry
781; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
782; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
783; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
784; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
785; GFX10-CU-NEXT:    ds_write_b32 v0, v1
786; GFX10-CU-NEXT:    s_endpgm
787;
788; SKIP-CACHE-INV-LABEL: local_singlethread_release_store:
789; SKIP-CACHE-INV:       ; %bb.0: ; %entry
790; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
791; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
792; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
793; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
794; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
795; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
796; SKIP-CACHE-INV-NEXT:    s_endpgm
797;
798; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_store:
799; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
800; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
801; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
802; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
803; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
804; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
805; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
806;
807; GFX90A-TGSPLIT-LABEL: local_singlethread_release_store:
808; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
809; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
810; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
811; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
812; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
813; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
814; GFX90A-TGSPLIT-NEXT:    s_endpgm
815;
816; GFX940-NOTTGSPLIT-LABEL: local_singlethread_release_store:
817; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
818; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
819; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
820; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
821; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
822; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
823; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
824;
825; GFX940-TGSPLIT-LABEL: local_singlethread_release_store:
826; GFX940-TGSPLIT:       ; %bb.0: ; %entry
827; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
828; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
829; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
830; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
831; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
832; GFX940-TGSPLIT-NEXT:    s_endpgm
833;
834; GFX11-WGP-LABEL: local_singlethread_release_store:
835; GFX11-WGP:       ; %bb.0: ; %entry
836; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
837; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
838; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
839; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
840; GFX11-WGP-NEXT:    s_endpgm
841;
842; GFX11-CU-LABEL: local_singlethread_release_store:
843; GFX11-CU:       ; %bb.0: ; %entry
844; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
845; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
846; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
847; GFX11-CU-NEXT:    ds_store_b32 v0, v1
848; GFX11-CU-NEXT:    s_endpgm
849    i32 %in, i32 addrspace(3)* %out) {
850entry:
851  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread") release, align 4
852  ret void
853}
854
855define amdgpu_kernel void @local_singlethread_seq_cst_store(
856; GFX6-LABEL: local_singlethread_seq_cst_store:
857; GFX6:       ; %bb.0: ; %entry
858; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
859; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
860; GFX6-NEXT:    s_mov_b32 m0, -1
861; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
862; GFX6-NEXT:    v_mov_b32_e32 v0, s0
863; GFX6-NEXT:    v_mov_b32_e32 v1, s1
864; GFX6-NEXT:    ds_write_b32 v0, v1
865; GFX6-NEXT:    s_endpgm
866;
867; GFX7-LABEL: local_singlethread_seq_cst_store:
868; GFX7:       ; %bb.0: ; %entry
869; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
870; GFX7-NEXT:    s_mov_b32 m0, -1
871; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
872; GFX7-NEXT:    v_mov_b32_e32 v0, s1
873; GFX7-NEXT:    v_mov_b32_e32 v1, s0
874; GFX7-NEXT:    ds_write_b32 v0, v1
875; GFX7-NEXT:    s_endpgm
876;
877; GFX10-WGP-LABEL: local_singlethread_seq_cst_store:
878; GFX10-WGP:       ; %bb.0: ; %entry
879; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
880; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
881; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
882; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
883; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
884; GFX10-WGP-NEXT:    s_endpgm
885;
886; GFX10-CU-LABEL: local_singlethread_seq_cst_store:
887; GFX10-CU:       ; %bb.0: ; %entry
888; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
889; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
890; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
891; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
892; GFX10-CU-NEXT:    ds_write_b32 v0, v1
893; GFX10-CU-NEXT:    s_endpgm
894;
895; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_store:
896; SKIP-CACHE-INV:       ; %bb.0: ; %entry
897; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
898; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
899; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
900; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
901; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
902; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
903; SKIP-CACHE-INV-NEXT:    s_endpgm
904;
905; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_store:
906; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
907; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
908; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
909; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
910; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
911; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
912; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
913;
914; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_store:
915; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
916; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
917; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
918; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
919; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
920; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
921; GFX90A-TGSPLIT-NEXT:    s_endpgm
922;
923; GFX940-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_store:
924; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
925; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
926; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
927; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
928; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
929; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
930; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
931;
932; GFX940-TGSPLIT-LABEL: local_singlethread_seq_cst_store:
933; GFX940-TGSPLIT:       ; %bb.0: ; %entry
934; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
935; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
936; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
937; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
938; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
939; GFX940-TGSPLIT-NEXT:    s_endpgm
940;
941; GFX11-WGP-LABEL: local_singlethread_seq_cst_store:
942; GFX11-WGP:       ; %bb.0: ; %entry
943; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
944; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
945; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
946; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
947; GFX11-WGP-NEXT:    s_endpgm
948;
949; GFX11-CU-LABEL: local_singlethread_seq_cst_store:
950; GFX11-CU:       ; %bb.0: ; %entry
951; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
952; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
953; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
954; GFX11-CU-NEXT:    ds_store_b32 v0, v1
955; GFX11-CU-NEXT:    s_endpgm
956    i32 %in, i32 addrspace(3)* %out) {
957entry:
958  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread") seq_cst, align 4
959  ret void
960}
961
962define amdgpu_kernel void @local_singlethread_monotonic_atomicrmw(
963; GFX6-LABEL: local_singlethread_monotonic_atomicrmw:
964; GFX6:       ; %bb.0: ; %entry
965; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
966; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
967; GFX6-NEXT:    s_mov_b32 m0, -1
968; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
969; GFX6-NEXT:    v_mov_b32_e32 v0, s0
970; GFX6-NEXT:    v_mov_b32_e32 v1, s1
971; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
972; GFX6-NEXT:    s_endpgm
973;
974; GFX7-LABEL: local_singlethread_monotonic_atomicrmw:
975; GFX7:       ; %bb.0: ; %entry
976; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
977; GFX7-NEXT:    s_mov_b32 m0, -1
978; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
979; GFX7-NEXT:    v_mov_b32_e32 v0, s0
980; GFX7-NEXT:    v_mov_b32_e32 v1, s1
981; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
982; GFX7-NEXT:    s_endpgm
983;
984; GFX10-WGP-LABEL: local_singlethread_monotonic_atomicrmw:
985; GFX10-WGP:       ; %bb.0: ; %entry
986; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
987; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
988; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
989; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
990; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
991; GFX10-WGP-NEXT:    s_endpgm
992;
993; GFX10-CU-LABEL: local_singlethread_monotonic_atomicrmw:
994; GFX10-CU:       ; %bb.0: ; %entry
995; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
996; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
997; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
998; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
999; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1000; GFX10-CU-NEXT:    s_endpgm
1001;
1002; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_atomicrmw:
1003; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1004; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1005; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1006; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1007; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1008; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1009; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1010; SKIP-CACHE-INV-NEXT:    s_endpgm
1011;
1012; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_atomicrmw:
1013; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1014; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1015; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1016; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1017; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1018; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1019; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1020;
1021; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_atomicrmw:
1022; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1023; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1024; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1025; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1026; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1027; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1028; GFX90A-TGSPLIT-NEXT:    s_endpgm
1029;
1030; GFX940-NOTTGSPLIT-LABEL: local_singlethread_monotonic_atomicrmw:
1031; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1032; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1033; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1034; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1035; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1036; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1037; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1038;
1039; GFX940-TGSPLIT-LABEL: local_singlethread_monotonic_atomicrmw:
1040; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1041; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1042; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1043; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1044; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1045; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1046; GFX940-TGSPLIT-NEXT:    s_endpgm
1047;
1048; GFX11-WGP-LABEL: local_singlethread_monotonic_atomicrmw:
1049; GFX11-WGP:       ; %bb.0: ; %entry
1050; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1051; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1052; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1053; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1054; GFX11-WGP-NEXT:    s_endpgm
1055;
1056; GFX11-CU-LABEL: local_singlethread_monotonic_atomicrmw:
1057; GFX11-CU:       ; %bb.0: ; %entry
1058; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1059; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1060; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1061; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1062; GFX11-CU-NEXT:    s_endpgm
1063    i32 addrspace(3)* %out, i32 %in) {
1064entry:
1065  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") monotonic
1066  ret void
1067}
1068
1069define amdgpu_kernel void @local_singlethread_acquire_atomicrmw(
1070; GFX6-LABEL: local_singlethread_acquire_atomicrmw:
1071; GFX6:       ; %bb.0: ; %entry
1072; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
1073; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
1074; GFX6-NEXT:    s_mov_b32 m0, -1
1075; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1076; GFX6-NEXT:    v_mov_b32_e32 v0, s0
1077; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1078; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1079; GFX6-NEXT:    s_endpgm
1080;
1081; GFX7-LABEL: local_singlethread_acquire_atomicrmw:
1082; GFX7:       ; %bb.0: ; %entry
1083; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1084; GFX7-NEXT:    s_mov_b32 m0, -1
1085; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1086; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1087; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1088; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1089; GFX7-NEXT:    s_endpgm
1090;
1091; GFX10-WGP-LABEL: local_singlethread_acquire_atomicrmw:
1092; GFX10-WGP:       ; %bb.0: ; %entry
1093; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1094; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1095; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1096; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1097; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1098; GFX10-WGP-NEXT:    s_endpgm
1099;
1100; GFX10-CU-LABEL: local_singlethread_acquire_atomicrmw:
1101; GFX10-CU:       ; %bb.0: ; %entry
1102; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1103; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1104; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1105; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1106; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1107; GFX10-CU-NEXT:    s_endpgm
1108;
1109; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_atomicrmw:
1110; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1111; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1112; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1113; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1114; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1115; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1116; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1117; SKIP-CACHE-INV-NEXT:    s_endpgm
1118;
1119; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_atomicrmw:
1120; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1121; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1122; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1123; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1124; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1125; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1126; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1127;
1128; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_atomicrmw:
1129; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1130; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1131; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1132; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1133; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1134; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1135; GFX90A-TGSPLIT-NEXT:    s_endpgm
1136;
1137; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acquire_atomicrmw:
1138; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1139; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1140; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1141; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1142; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1143; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1144; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1145;
1146; GFX940-TGSPLIT-LABEL: local_singlethread_acquire_atomicrmw:
1147; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1148; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1149; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1150; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1151; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1152; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1153; GFX940-TGSPLIT-NEXT:    s_endpgm
1154;
1155; GFX11-WGP-LABEL: local_singlethread_acquire_atomicrmw:
1156; GFX11-WGP:       ; %bb.0: ; %entry
1157; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1158; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1159; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1160; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1161; GFX11-WGP-NEXT:    s_endpgm
1162;
1163; GFX11-CU-LABEL: local_singlethread_acquire_atomicrmw:
1164; GFX11-CU:       ; %bb.0: ; %entry
1165; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1166; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1167; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1168; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1169; GFX11-CU-NEXT:    s_endpgm
1170    i32 addrspace(3)* %out, i32 %in) {
1171entry:
1172  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") acquire
1173  ret void
1174}
1175
1176define amdgpu_kernel void @local_singlethread_release_atomicrmw(
1177; GFX6-LABEL: local_singlethread_release_atomicrmw:
1178; GFX6:       ; %bb.0: ; %entry
1179; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
1180; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
1181; GFX6-NEXT:    s_mov_b32 m0, -1
1182; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1183; GFX6-NEXT:    v_mov_b32_e32 v0, s0
1184; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1185; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1186; GFX6-NEXT:    s_endpgm
1187;
1188; GFX7-LABEL: local_singlethread_release_atomicrmw:
1189; GFX7:       ; %bb.0: ; %entry
1190; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1191; GFX7-NEXT:    s_mov_b32 m0, -1
1192; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1193; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1194; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1195; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1196; GFX7-NEXT:    s_endpgm
1197;
1198; GFX10-WGP-LABEL: local_singlethread_release_atomicrmw:
1199; GFX10-WGP:       ; %bb.0: ; %entry
1200; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1201; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1202; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1203; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1204; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1205; GFX10-WGP-NEXT:    s_endpgm
1206;
1207; GFX10-CU-LABEL: local_singlethread_release_atomicrmw:
1208; GFX10-CU:       ; %bb.0: ; %entry
1209; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1210; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1211; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1212; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1213; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1214; GFX10-CU-NEXT:    s_endpgm
1215;
1216; SKIP-CACHE-INV-LABEL: local_singlethread_release_atomicrmw:
1217; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1218; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1219; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1220; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1221; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1222; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1223; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1224; SKIP-CACHE-INV-NEXT:    s_endpgm
1225;
1226; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_atomicrmw:
1227; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1228; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1229; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1230; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1231; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1232; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1233; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1234;
1235; GFX90A-TGSPLIT-LABEL: local_singlethread_release_atomicrmw:
1236; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1237; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1238; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1239; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1240; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1241; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1242; GFX90A-TGSPLIT-NEXT:    s_endpgm
1243;
1244; GFX940-NOTTGSPLIT-LABEL: local_singlethread_release_atomicrmw:
1245; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1246; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1247; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1248; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1249; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1250; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1251; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1252;
1253; GFX940-TGSPLIT-LABEL: local_singlethread_release_atomicrmw:
1254; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1255; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1256; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1257; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1258; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1259; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1260; GFX940-TGSPLIT-NEXT:    s_endpgm
1261;
1262; GFX11-WGP-LABEL: local_singlethread_release_atomicrmw:
1263; GFX11-WGP:       ; %bb.0: ; %entry
1264; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1265; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1266; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1267; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1268; GFX11-WGP-NEXT:    s_endpgm
1269;
1270; GFX11-CU-LABEL: local_singlethread_release_atomicrmw:
1271; GFX11-CU:       ; %bb.0: ; %entry
1272; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1273; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1274; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1275; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1276; GFX11-CU-NEXT:    s_endpgm
1277    i32 addrspace(3)* %out, i32 %in) {
1278entry:
1279  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") release
1280  ret void
1281}
1282
1283define amdgpu_kernel void @local_singlethread_acq_rel_atomicrmw(
1284; GFX6-LABEL: local_singlethread_acq_rel_atomicrmw:
1285; GFX6:       ; %bb.0: ; %entry
1286; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
1287; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
1288; GFX6-NEXT:    s_mov_b32 m0, -1
1289; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1290; GFX6-NEXT:    v_mov_b32_e32 v0, s0
1291; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1292; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1293; GFX6-NEXT:    s_endpgm
1294;
1295; GFX7-LABEL: local_singlethread_acq_rel_atomicrmw:
1296; GFX7:       ; %bb.0: ; %entry
1297; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1298; GFX7-NEXT:    s_mov_b32 m0, -1
1299; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1300; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1301; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1302; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1303; GFX7-NEXT:    s_endpgm
1304;
1305; GFX10-WGP-LABEL: local_singlethread_acq_rel_atomicrmw:
1306; GFX10-WGP:       ; %bb.0: ; %entry
1307; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1308; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1309; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1310; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1311; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1312; GFX10-WGP-NEXT:    s_endpgm
1313;
1314; GFX10-CU-LABEL: local_singlethread_acq_rel_atomicrmw:
1315; GFX10-CU:       ; %bb.0: ; %entry
1316; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1317; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1318; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1319; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1320; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1321; GFX10-CU-NEXT:    s_endpgm
1322;
1323; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_atomicrmw:
1324; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1325; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1326; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1327; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1328; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1329; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1330; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1331; SKIP-CACHE-INV-NEXT:    s_endpgm
1332;
1333; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_atomicrmw:
1334; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1335; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1336; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1337; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1338; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1339; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1340; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1341;
1342; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_atomicrmw:
1343; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1344; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1345; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1346; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1347; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1348; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1349; GFX90A-TGSPLIT-NEXT:    s_endpgm
1350;
1351; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_atomicrmw:
1352; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1353; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1354; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1355; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1356; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1357; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1358; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1359;
1360; GFX940-TGSPLIT-LABEL: local_singlethread_acq_rel_atomicrmw:
1361; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1362; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1363; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1364; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1365; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1366; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1367; GFX940-TGSPLIT-NEXT:    s_endpgm
1368;
1369; GFX11-WGP-LABEL: local_singlethread_acq_rel_atomicrmw:
1370; GFX11-WGP:       ; %bb.0: ; %entry
1371; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1372; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1373; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1374; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1375; GFX11-WGP-NEXT:    s_endpgm
1376;
1377; GFX11-CU-LABEL: local_singlethread_acq_rel_atomicrmw:
1378; GFX11-CU:       ; %bb.0: ; %entry
1379; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1380; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1381; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1382; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1383; GFX11-CU-NEXT:    s_endpgm
1384    i32 addrspace(3)* %out, i32 %in) {
1385entry:
1386  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") acq_rel
1387  ret void
1388}
1389
1390define amdgpu_kernel void @local_singlethread_seq_cst_atomicrmw(
1391; GFX6-LABEL: local_singlethread_seq_cst_atomicrmw:
1392; GFX6:       ; %bb.0: ; %entry
1393; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
1394; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
1395; GFX6-NEXT:    s_mov_b32 m0, -1
1396; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1397; GFX6-NEXT:    v_mov_b32_e32 v0, s0
1398; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1399; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1400; GFX6-NEXT:    s_endpgm
1401;
1402; GFX7-LABEL: local_singlethread_seq_cst_atomicrmw:
1403; GFX7:       ; %bb.0: ; %entry
1404; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1405; GFX7-NEXT:    s_mov_b32 m0, -1
1406; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1407; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1408; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1409; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1410; GFX7-NEXT:    s_endpgm
1411;
1412; GFX10-WGP-LABEL: local_singlethread_seq_cst_atomicrmw:
1413; GFX10-WGP:       ; %bb.0: ; %entry
1414; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1415; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1416; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1417; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1418; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1419; GFX10-WGP-NEXT:    s_endpgm
1420;
1421; GFX10-CU-LABEL: local_singlethread_seq_cst_atomicrmw:
1422; GFX10-CU:       ; %bb.0: ; %entry
1423; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1424; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1425; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1426; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1427; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1428; GFX10-CU-NEXT:    s_endpgm
1429;
1430; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_atomicrmw:
1431; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1432; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1433; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1434; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1435; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1436; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1437; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1438; SKIP-CACHE-INV-NEXT:    s_endpgm
1439;
1440; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_atomicrmw:
1441; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1442; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1443; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1444; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1445; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1446; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1447; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1448;
1449; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_atomicrmw:
1450; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1451; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1452; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1453; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1454; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1455; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1456; GFX90A-TGSPLIT-NEXT:    s_endpgm
1457;
1458; GFX940-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_atomicrmw:
1459; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1460; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1461; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1462; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1463; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1464; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1465; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1466;
1467; GFX940-TGSPLIT-LABEL: local_singlethread_seq_cst_atomicrmw:
1468; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1469; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1470; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1471; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1472; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1473; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1474; GFX940-TGSPLIT-NEXT:    s_endpgm
1475;
1476; GFX11-WGP-LABEL: local_singlethread_seq_cst_atomicrmw:
1477; GFX11-WGP:       ; %bb.0: ; %entry
1478; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1479; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1480; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1481; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1482; GFX11-WGP-NEXT:    s_endpgm
1483;
1484; GFX11-CU-LABEL: local_singlethread_seq_cst_atomicrmw:
1485; GFX11-CU:       ; %bb.0: ; %entry
1486; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1487; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1488; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1489; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
1490; GFX11-CU-NEXT:    s_endpgm
1491    i32 addrspace(3)* %out, i32 %in) {
1492entry:
1493  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") seq_cst
1494  ret void
1495}
1496
1497define amdgpu_kernel void @local_singlethread_acquire_ret_atomicrmw(
1498; GFX6-LABEL: local_singlethread_acquire_ret_atomicrmw:
1499; GFX6:       ; %bb.0: ; %entry
1500; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
1501; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
1502; GFX6-NEXT:    s_mov_b32 m0, -1
1503; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1504; GFX6-NEXT:    v_mov_b32_e32 v0, s0
1505; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1506; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1507; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1508; GFX6-NEXT:    ds_write_b32 v0, v1
1509; GFX6-NEXT:    s_endpgm
1510;
1511; GFX7-LABEL: local_singlethread_acquire_ret_atomicrmw:
1512; GFX7:       ; %bb.0: ; %entry
1513; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1514; GFX7-NEXT:    s_mov_b32 m0, -1
1515; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1516; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1517; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1518; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1519; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1520; GFX7-NEXT:    ds_write_b32 v0, v1
1521; GFX7-NEXT:    s_endpgm
1522;
1523; GFX10-WGP-LABEL: local_singlethread_acquire_ret_atomicrmw:
1524; GFX10-WGP:       ; %bb.0: ; %entry
1525; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1526; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1527; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1528; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1529; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1530; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1531; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
1532; GFX10-WGP-NEXT:    s_endpgm
1533;
1534; GFX10-CU-LABEL: local_singlethread_acquire_ret_atomicrmw:
1535; GFX10-CU:       ; %bb.0: ; %entry
1536; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1537; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1538; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1539; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1540; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1541; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1542; GFX10-CU-NEXT:    ds_write_b32 v0, v1
1543; GFX10-CU-NEXT:    s_endpgm
1544;
1545; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_ret_atomicrmw:
1546; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1547; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1548; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1549; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1550; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1551; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1552; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1553; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1554; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
1555; SKIP-CACHE-INV-NEXT:    s_endpgm
1556;
1557; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_ret_atomicrmw:
1558; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1559; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1560; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1561; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1562; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1563; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1564; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1565; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
1566; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1567;
1568; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_ret_atomicrmw:
1569; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1570; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1571; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1572; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1573; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1574; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1575; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1576; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
1577; GFX90A-TGSPLIT-NEXT:    s_endpgm
1578;
1579; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acquire_ret_atomicrmw:
1580; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1581; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1582; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1583; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1584; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1585; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1586; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1587; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
1588; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1589;
1590; GFX940-TGSPLIT-LABEL: local_singlethread_acquire_ret_atomicrmw:
1591; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1592; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1593; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1594; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1595; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1596; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1597; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1598; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
1599; GFX940-TGSPLIT-NEXT:    s_endpgm
1600;
1601; GFX11-WGP-LABEL: local_singlethread_acquire_ret_atomicrmw:
1602; GFX11-WGP:       ; %bb.0: ; %entry
1603; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1604; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1605; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1606; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
1607; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1608; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
1609; GFX11-WGP-NEXT:    s_endpgm
1610;
1611; GFX11-CU-LABEL: local_singlethread_acquire_ret_atomicrmw:
1612; GFX11-CU:       ; %bb.0: ; %entry
1613; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1614; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1615; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1616; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
1617; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1618; GFX11-CU-NEXT:    ds_store_b32 v0, v1
1619; GFX11-CU-NEXT:    s_endpgm
1620    i32 addrspace(3)* %out, i32 %in) {
1621entry:
1622  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") acquire
1623  store i32 %val, i32 addrspace(3)* %out, align 4
1624  ret void
1625}
1626
1627define amdgpu_kernel void @local_singlethread_acq_rel_ret_atomicrmw(
1628; GFX6-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1629; GFX6:       ; %bb.0: ; %entry
1630; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
1631; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
1632; GFX6-NEXT:    s_mov_b32 m0, -1
1633; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1634; GFX6-NEXT:    v_mov_b32_e32 v0, s0
1635; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1636; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1637; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1638; GFX6-NEXT:    ds_write_b32 v0, v1
1639; GFX6-NEXT:    s_endpgm
1640;
1641; GFX7-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1642; GFX7:       ; %bb.0: ; %entry
1643; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1644; GFX7-NEXT:    s_mov_b32 m0, -1
1645; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1646; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1647; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1648; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1649; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1650; GFX7-NEXT:    ds_write_b32 v0, v1
1651; GFX7-NEXT:    s_endpgm
1652;
1653; GFX10-WGP-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1654; GFX10-WGP:       ; %bb.0: ; %entry
1655; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1656; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1657; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1658; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1659; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1660; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1661; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
1662; GFX10-WGP-NEXT:    s_endpgm
1663;
1664; GFX10-CU-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1665; GFX10-CU:       ; %bb.0: ; %entry
1666; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1667; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1668; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1669; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1670; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1671; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1672; GFX10-CU-NEXT:    ds_write_b32 v0, v1
1673; GFX10-CU-NEXT:    s_endpgm
1674;
1675; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1676; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1677; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1678; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1679; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1680; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1681; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1682; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1683; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1684; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
1685; SKIP-CACHE-INV-NEXT:    s_endpgm
1686;
1687; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1688; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1689; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1690; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1691; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1692; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1693; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1694; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1695; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
1696; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1697;
1698; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1699; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1700; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1701; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1702; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1703; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1704; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1705; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1706; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
1707; GFX90A-TGSPLIT-NEXT:    s_endpgm
1708;
1709; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1710; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1711; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1712; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1713; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1714; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1715; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1716; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1717; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
1718; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1719;
1720; GFX940-TGSPLIT-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1721; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1722; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1723; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1724; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1725; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1726; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1727; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1728; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
1729; GFX940-TGSPLIT-NEXT:    s_endpgm
1730;
1731; GFX11-WGP-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1732; GFX11-WGP:       ; %bb.0: ; %entry
1733; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1734; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1735; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1736; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
1737; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1738; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
1739; GFX11-WGP-NEXT:    s_endpgm
1740;
1741; GFX11-CU-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1742; GFX11-CU:       ; %bb.0: ; %entry
1743; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1744; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1745; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1746; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
1747; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1748; GFX11-CU-NEXT:    ds_store_b32 v0, v1
1749; GFX11-CU-NEXT:    s_endpgm
1750    i32 addrspace(3)* %out, i32 %in) {
1751entry:
1752  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") acq_rel
1753  store i32 %val, i32 addrspace(3)* %out, align 4
1754  ret void
1755}
1756
1757define amdgpu_kernel void @local_singlethread_seq_cst_ret_atomicrmw(
1758; GFX6-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1759; GFX6:       ; %bb.0: ; %entry
1760; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
1761; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
1762; GFX6-NEXT:    s_mov_b32 m0, -1
1763; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1764; GFX6-NEXT:    v_mov_b32_e32 v0, s0
1765; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1766; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1767; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1768; GFX6-NEXT:    ds_write_b32 v0, v1
1769; GFX6-NEXT:    s_endpgm
1770;
1771; GFX7-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1772; GFX7:       ; %bb.0: ; %entry
1773; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1774; GFX7-NEXT:    s_mov_b32 m0, -1
1775; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1776; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1777; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1778; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1779; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1780; GFX7-NEXT:    ds_write_b32 v0, v1
1781; GFX7-NEXT:    s_endpgm
1782;
1783; GFX10-WGP-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1784; GFX10-WGP:       ; %bb.0: ; %entry
1785; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1786; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1787; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1788; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1789; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1790; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1791; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
1792; GFX10-WGP-NEXT:    s_endpgm
1793;
1794; GFX10-CU-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1795; GFX10-CU:       ; %bb.0: ; %entry
1796; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1797; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1798; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1799; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1800; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1801; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1802; GFX10-CU-NEXT:    ds_write_b32 v0, v1
1803; GFX10-CU-NEXT:    s_endpgm
1804;
1805; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1806; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1807; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1808; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1809; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1810; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1811; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1812; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1813; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1814; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
1815; SKIP-CACHE-INV-NEXT:    s_endpgm
1816;
1817; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1818; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1819; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1820; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1821; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1822; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1823; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1824; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1825; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
1826; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1827;
1828; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1829; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1830; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1831; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1832; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1833; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1834; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1835; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1836; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
1837; GFX90A-TGSPLIT-NEXT:    s_endpgm
1838;
1839; GFX940-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1840; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1841; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1842; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1843; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1844; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1845; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1846; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1847; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
1848; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1849;
1850; GFX940-TGSPLIT-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1851; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1852; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1853; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1854; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1855; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1856; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1857; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1858; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
1859; GFX940-TGSPLIT-NEXT:    s_endpgm
1860;
1861; GFX11-WGP-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1862; GFX11-WGP:       ; %bb.0: ; %entry
1863; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1864; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1865; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1866; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
1867; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1868; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
1869; GFX11-WGP-NEXT:    s_endpgm
1870;
1871; GFX11-CU-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1872; GFX11-CU:       ; %bb.0: ; %entry
1873; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1874; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1875; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1876; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
1877; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1878; GFX11-CU-NEXT:    ds_store_b32 v0, v1
1879; GFX11-CU-NEXT:    s_endpgm
1880    i32 addrspace(3)* %out, i32 %in) {
1881entry:
1882  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") seq_cst
1883  store i32 %val, i32 addrspace(3)* %out, align 4
1884  ret void
1885}
1886
1887define amdgpu_kernel void @local_singlethread_monotonic_monotonic_cmpxchg(
1888; GFX6-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1889; GFX6:       ; %bb.0: ; %entry
1890; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
1891; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
1892; GFX6-NEXT:    s_mov_b32 m0, -1
1893; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1894; GFX6-NEXT:    v_mov_b32_e32 v0, s2
1895; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1896; GFX6-NEXT:    v_mov_b32_e32 v2, s0
1897; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1898; GFX6-NEXT:    s_endpgm
1899;
1900; GFX7-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1901; GFX7:       ; %bb.0: ; %entry
1902; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1903; GFX7-NEXT:    s_mov_b32 m0, -1
1904; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1905; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1906; GFX7-NEXT:    v_mov_b32_e32 v1, s2
1907; GFX7-NEXT:    v_mov_b32_e32 v2, s1
1908; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1909; GFX7-NEXT:    s_endpgm
1910;
1911; GFX10-WGP-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1912; GFX10-WGP:       ; %bb.0: ; %entry
1913; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1914; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1915; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1916; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1917; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
1918; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1919; GFX10-WGP-NEXT:    s_endpgm
1920;
1921; GFX10-CU-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1922; GFX10-CU:       ; %bb.0: ; %entry
1923; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1924; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1925; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1926; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1927; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
1928; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1929; GFX10-CU-NEXT:    s_endpgm
1930;
1931; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1932; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1933; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
1934; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1935; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1936; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1937; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
1938; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
1939; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1940; SKIP-CACHE-INV-NEXT:    s_endpgm
1941;
1942; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1943; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1944; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1945; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1946; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1947; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1948; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1949; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1950; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1951;
1952; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1953; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1954; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1955; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1956; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1957; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1958; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1959; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1960; GFX90A-TGSPLIT-NEXT:    s_endpgm
1961;
1962; GFX940-NOTTGSPLIT-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1963; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1964; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
1965; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1966; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1967; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1968; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1969; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1970; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1971;
1972; GFX940-TGSPLIT-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1973; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1974; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
1975; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1976; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1977; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1978; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1979; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1980; GFX940-TGSPLIT-NEXT:    s_endpgm
1981;
1982; GFX11-WGP-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1983; GFX11-WGP:       ; %bb.0: ; %entry
1984; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
1985; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1986; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1987; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
1988; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
1989; GFX11-WGP-NEXT:    s_endpgm
1990;
1991; GFX11-CU-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1992; GFX11-CU:       ; %bb.0: ; %entry
1993; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
1994; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1995; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
1996; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
1997; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
1998; GFX11-CU-NEXT:    s_endpgm
1999    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2000entry:
2001  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2002  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
2003  ret void
2004}
2005
2006define amdgpu_kernel void @local_singlethread_acquire_monotonic_cmpxchg(
2007; GFX6-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2008; GFX6:       ; %bb.0: ; %entry
2009; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2010; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2011; GFX6-NEXT:    s_mov_b32 m0, -1
2012; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2013; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2014; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2015; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2016; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2017; GFX6-NEXT:    s_endpgm
2018;
2019; GFX7-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2020; GFX7:       ; %bb.0: ; %entry
2021; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2022; GFX7-NEXT:    s_mov_b32 m0, -1
2023; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2024; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2025; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2026; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2027; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2028; GFX7-NEXT:    s_endpgm
2029;
2030; GFX10-WGP-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2031; GFX10-WGP:       ; %bb.0: ; %entry
2032; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2033; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2034; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2035; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2036; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2037; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2038; GFX10-WGP-NEXT:    s_endpgm
2039;
2040; GFX10-CU-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2041; GFX10-CU:       ; %bb.0: ; %entry
2042; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2043; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2044; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2045; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2046; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2047; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2048; GFX10-CU-NEXT:    s_endpgm
2049;
2050; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2051; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2052; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2053; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2054; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2055; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2056; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2057; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2058; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2059; SKIP-CACHE-INV-NEXT:    s_endpgm
2060;
2061; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2062; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2063; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2064; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2065; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2066; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2067; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2068; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2069; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2070;
2071; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2072; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2073; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2074; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2075; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2076; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2077; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2078; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2079; GFX90A-TGSPLIT-NEXT:    s_endpgm
2080;
2081; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2082; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2083; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2084; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2085; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2086; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2087; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2088; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2089; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2090;
2091; GFX940-TGSPLIT-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2092; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2093; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2094; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2095; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2096; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2097; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2098; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2099; GFX940-TGSPLIT-NEXT:    s_endpgm
2100;
2101; GFX11-WGP-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2102; GFX11-WGP:       ; %bb.0: ; %entry
2103; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2104; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2105; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2106; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
2107; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2108; GFX11-WGP-NEXT:    s_endpgm
2109;
2110; GFX11-CU-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
2111; GFX11-CU:       ; %bb.0: ; %entry
2112; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2113; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2114; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2115; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2116; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2117; GFX11-CU-NEXT:    s_endpgm
2118    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2119entry:
2120  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2121  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
2122  ret void
2123}
2124
2125define amdgpu_kernel void @local_singlethread_release_monotonic_cmpxchg(
2126; GFX6-LABEL: local_singlethread_release_monotonic_cmpxchg:
2127; GFX6:       ; %bb.0: ; %entry
2128; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2129; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2130; GFX6-NEXT:    s_mov_b32 m0, -1
2131; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2132; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2133; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2134; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2135; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2136; GFX6-NEXT:    s_endpgm
2137;
2138; GFX7-LABEL: local_singlethread_release_monotonic_cmpxchg:
2139; GFX7:       ; %bb.0: ; %entry
2140; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2141; GFX7-NEXT:    s_mov_b32 m0, -1
2142; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2143; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2144; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2145; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2146; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2147; GFX7-NEXT:    s_endpgm
2148;
2149; GFX10-WGP-LABEL: local_singlethread_release_monotonic_cmpxchg:
2150; GFX10-WGP:       ; %bb.0: ; %entry
2151; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2152; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2153; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2154; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2155; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2156; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2157; GFX10-WGP-NEXT:    s_endpgm
2158;
2159; GFX10-CU-LABEL: local_singlethread_release_monotonic_cmpxchg:
2160; GFX10-CU:       ; %bb.0: ; %entry
2161; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2162; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2163; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2164; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2165; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2166; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2167; GFX10-CU-NEXT:    s_endpgm
2168;
2169; SKIP-CACHE-INV-LABEL: local_singlethread_release_monotonic_cmpxchg:
2170; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2171; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2172; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2173; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2174; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2175; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2176; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2177; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2178; SKIP-CACHE-INV-NEXT:    s_endpgm
2179;
2180; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_monotonic_cmpxchg:
2181; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2182; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2183; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2184; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2185; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2186; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2187; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2188; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2189;
2190; GFX90A-TGSPLIT-LABEL: local_singlethread_release_monotonic_cmpxchg:
2191; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2192; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2193; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2194; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2195; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2196; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2197; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2198; GFX90A-TGSPLIT-NEXT:    s_endpgm
2199;
2200; GFX940-NOTTGSPLIT-LABEL: local_singlethread_release_monotonic_cmpxchg:
2201; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2202; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2203; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2204; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2205; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2206; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2207; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2208; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2209;
2210; GFX940-TGSPLIT-LABEL: local_singlethread_release_monotonic_cmpxchg:
2211; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2212; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2213; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2214; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2215; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2216; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2217; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2218; GFX940-TGSPLIT-NEXT:    s_endpgm
2219;
2220; GFX11-WGP-LABEL: local_singlethread_release_monotonic_cmpxchg:
2221; GFX11-WGP:       ; %bb.0: ; %entry
2222; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2223; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2224; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2225; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
2226; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2227; GFX11-WGP-NEXT:    s_endpgm
2228;
2229; GFX11-CU-LABEL: local_singlethread_release_monotonic_cmpxchg:
2230; GFX11-CU:       ; %bb.0: ; %entry
2231; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2232; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2233; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2234; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2235; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2236; GFX11-CU-NEXT:    s_endpgm
2237    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2238entry:
2239  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2240  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
2241  ret void
2242}
2243
2244define amdgpu_kernel void @local_singlethread_acq_rel_monotonic_cmpxchg(
2245; GFX6-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2246; GFX6:       ; %bb.0: ; %entry
2247; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2248; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2249; GFX6-NEXT:    s_mov_b32 m0, -1
2250; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2251; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2252; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2253; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2254; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2255; GFX6-NEXT:    s_endpgm
2256;
2257; GFX7-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2258; GFX7:       ; %bb.0: ; %entry
2259; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2260; GFX7-NEXT:    s_mov_b32 m0, -1
2261; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2262; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2263; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2264; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2265; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2266; GFX7-NEXT:    s_endpgm
2267;
2268; GFX10-WGP-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2269; GFX10-WGP:       ; %bb.0: ; %entry
2270; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2271; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2272; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2273; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2274; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2275; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2276; GFX10-WGP-NEXT:    s_endpgm
2277;
2278; GFX10-CU-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2279; GFX10-CU:       ; %bb.0: ; %entry
2280; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2281; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2282; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2283; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2284; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2285; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2286; GFX10-CU-NEXT:    s_endpgm
2287;
2288; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2289; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2290; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2291; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2292; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2293; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2294; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2295; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2296; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2297; SKIP-CACHE-INV-NEXT:    s_endpgm
2298;
2299; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2300; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2301; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2302; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2303; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2304; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2305; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2306; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2307; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2308;
2309; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2310; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2311; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2312; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2313; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2314; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2315; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2316; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2317; GFX90A-TGSPLIT-NEXT:    s_endpgm
2318;
2319; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2320; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2321; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2322; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2323; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2324; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2325; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2326; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2327; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2328;
2329; GFX940-TGSPLIT-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2330; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2331; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2332; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2333; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2334; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2335; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2336; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2337; GFX940-TGSPLIT-NEXT:    s_endpgm
2338;
2339; GFX11-WGP-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2340; GFX11-WGP:       ; %bb.0: ; %entry
2341; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2342; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2343; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2344; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
2345; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2346; GFX11-WGP-NEXT:    s_endpgm
2347;
2348; GFX11-CU-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
2349; GFX11-CU:       ; %bb.0: ; %entry
2350; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2351; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2352; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2353; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2354; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2355; GFX11-CU-NEXT:    s_endpgm
2356    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2357entry:
2358  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2359  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
2360  ret void
2361}
2362
2363define amdgpu_kernel void @local_singlethread_seq_cst_monotonic_cmpxchg(
2364; GFX6-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2365; GFX6:       ; %bb.0: ; %entry
2366; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2367; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2368; GFX6-NEXT:    s_mov_b32 m0, -1
2369; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2370; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2371; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2372; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2373; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2374; GFX6-NEXT:    s_endpgm
2375;
2376; GFX7-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2377; GFX7:       ; %bb.0: ; %entry
2378; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2379; GFX7-NEXT:    s_mov_b32 m0, -1
2380; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2381; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2382; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2383; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2384; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2385; GFX7-NEXT:    s_endpgm
2386;
2387; GFX10-WGP-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2388; GFX10-WGP:       ; %bb.0: ; %entry
2389; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2390; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2391; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2392; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2393; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2394; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2395; GFX10-WGP-NEXT:    s_endpgm
2396;
2397; GFX10-CU-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2398; GFX10-CU:       ; %bb.0: ; %entry
2399; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2400; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2401; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2402; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2403; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2404; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2405; GFX10-CU-NEXT:    s_endpgm
2406;
2407; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2408; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2409; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2410; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2411; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2412; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2413; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2414; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2415; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2416; SKIP-CACHE-INV-NEXT:    s_endpgm
2417;
2418; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2419; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2420; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2421; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2422; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2423; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2424; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2425; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2426; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2427;
2428; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2429; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2430; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2431; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2432; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2433; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2434; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2435; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2436; GFX90A-TGSPLIT-NEXT:    s_endpgm
2437;
2438; GFX940-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2439; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2440; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2441; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2442; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2443; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2444; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2445; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2446; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2447;
2448; GFX940-TGSPLIT-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2449; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2450; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2451; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2452; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2453; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2454; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2455; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2456; GFX940-TGSPLIT-NEXT:    s_endpgm
2457;
2458; GFX11-WGP-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2459; GFX11-WGP:       ; %bb.0: ; %entry
2460; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2461; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2462; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2463; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
2464; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2465; GFX11-WGP-NEXT:    s_endpgm
2466;
2467; GFX11-CU-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
2468; GFX11-CU:       ; %bb.0: ; %entry
2469; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2470; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2471; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2472; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2473; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2474; GFX11-CU-NEXT:    s_endpgm
2475    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2476entry:
2477  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2478  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
2479  ret void
2480}
2481
2482define amdgpu_kernel void @local_singlethread_monotonic_acquire_cmpxchg(
2483; GFX6-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2484; GFX6:       ; %bb.0: ; %entry
2485; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2486; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2487; GFX6-NEXT:    s_mov_b32 m0, -1
2488; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2489; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2490; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2491; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2492; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2493; GFX6-NEXT:    s_endpgm
2494;
2495; GFX7-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2496; GFX7:       ; %bb.0: ; %entry
2497; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2498; GFX7-NEXT:    s_mov_b32 m0, -1
2499; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2500; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2501; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2502; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2503; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2504; GFX7-NEXT:    s_endpgm
2505;
2506; GFX10-WGP-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2507; GFX10-WGP:       ; %bb.0: ; %entry
2508; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2509; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2510; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2511; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2512; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2513; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2514; GFX10-WGP-NEXT:    s_endpgm
2515;
2516; GFX10-CU-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2517; GFX10-CU:       ; %bb.0: ; %entry
2518; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2519; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2520; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2521; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2522; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2523; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2524; GFX10-CU-NEXT:    s_endpgm
2525;
2526; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2527; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2528; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2529; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2530; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2531; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2532; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2533; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2534; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2535; SKIP-CACHE-INV-NEXT:    s_endpgm
2536;
2537; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2538; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2539; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2540; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2541; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2542; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2543; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2544; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2545; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2546;
2547; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2548; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2549; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2550; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2551; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2552; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2553; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2554; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2555; GFX90A-TGSPLIT-NEXT:    s_endpgm
2556;
2557; GFX940-NOTTGSPLIT-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2558; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2559; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2560; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2561; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2562; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2563; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2564; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2565; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2566;
2567; GFX940-TGSPLIT-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2568; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2569; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2570; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2571; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2572; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2573; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2574; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2575; GFX940-TGSPLIT-NEXT:    s_endpgm
2576;
2577; GFX11-WGP-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2578; GFX11-WGP:       ; %bb.0: ; %entry
2579; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2580; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2581; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2582; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
2583; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2584; GFX11-WGP-NEXT:    s_endpgm
2585;
2586; GFX11-CU-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
2587; GFX11-CU:       ; %bb.0: ; %entry
2588; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2589; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2590; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2591; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2592; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2593; GFX11-CU-NEXT:    s_endpgm
2594    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2595entry:
2596  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2597  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic acquire
2598  ret void
2599}
2600
2601define amdgpu_kernel void @local_singlethread_acquire_acquire_cmpxchg(
2602; GFX6-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2603; GFX6:       ; %bb.0: ; %entry
2604; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2605; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2606; GFX6-NEXT:    s_mov_b32 m0, -1
2607; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2608; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2609; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2610; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2611; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2612; GFX6-NEXT:    s_endpgm
2613;
2614; GFX7-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2615; GFX7:       ; %bb.0: ; %entry
2616; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2617; GFX7-NEXT:    s_mov_b32 m0, -1
2618; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2619; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2620; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2621; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2622; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2623; GFX7-NEXT:    s_endpgm
2624;
2625; GFX10-WGP-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2626; GFX10-WGP:       ; %bb.0: ; %entry
2627; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2628; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2629; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2630; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2631; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2632; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2633; GFX10-WGP-NEXT:    s_endpgm
2634;
2635; GFX10-CU-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2636; GFX10-CU:       ; %bb.0: ; %entry
2637; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2638; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2639; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2640; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2641; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2642; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2643; GFX10-CU-NEXT:    s_endpgm
2644;
2645; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2646; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2647; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2648; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2649; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2650; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2651; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2652; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2653; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2654; SKIP-CACHE-INV-NEXT:    s_endpgm
2655;
2656; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2657; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2658; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2659; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2660; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2661; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2662; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2663; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2664; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2665;
2666; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2667; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2668; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2669; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2670; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2671; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2672; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2673; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2674; GFX90A-TGSPLIT-NEXT:    s_endpgm
2675;
2676; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2677; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2678; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2679; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2680; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2681; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2682; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2683; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2684; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2685;
2686; GFX940-TGSPLIT-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2687; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2688; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2689; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2690; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2691; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2692; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2693; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2694; GFX940-TGSPLIT-NEXT:    s_endpgm
2695;
2696; GFX11-WGP-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2697; GFX11-WGP:       ; %bb.0: ; %entry
2698; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2699; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2700; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2701; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
2702; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2703; GFX11-WGP-NEXT:    s_endpgm
2704;
2705; GFX11-CU-LABEL: local_singlethread_acquire_acquire_cmpxchg:
2706; GFX11-CU:       ; %bb.0: ; %entry
2707; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2708; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2709; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2710; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2711; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2712; GFX11-CU-NEXT:    s_endpgm
2713    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2714entry:
2715  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2716  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
2717  ret void
2718}
2719
2720define amdgpu_kernel void @local_singlethread_release_acquire_cmpxchg(
2721; GFX6-LABEL: local_singlethread_release_acquire_cmpxchg:
2722; GFX6:       ; %bb.0: ; %entry
2723; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2724; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2725; GFX6-NEXT:    s_mov_b32 m0, -1
2726; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2727; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2728; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2729; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2730; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2731; GFX6-NEXT:    s_endpgm
2732;
2733; GFX7-LABEL: local_singlethread_release_acquire_cmpxchg:
2734; GFX7:       ; %bb.0: ; %entry
2735; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2736; GFX7-NEXT:    s_mov_b32 m0, -1
2737; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2738; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2739; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2740; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2741; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2742; GFX7-NEXT:    s_endpgm
2743;
2744; GFX10-WGP-LABEL: local_singlethread_release_acquire_cmpxchg:
2745; GFX10-WGP:       ; %bb.0: ; %entry
2746; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2747; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2748; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2749; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2750; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2751; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2752; GFX10-WGP-NEXT:    s_endpgm
2753;
2754; GFX10-CU-LABEL: local_singlethread_release_acquire_cmpxchg:
2755; GFX10-CU:       ; %bb.0: ; %entry
2756; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2757; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2758; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2759; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2760; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2761; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2762; GFX10-CU-NEXT:    s_endpgm
2763;
2764; SKIP-CACHE-INV-LABEL: local_singlethread_release_acquire_cmpxchg:
2765; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2766; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2767; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2768; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2769; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2770; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2771; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2772; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2773; SKIP-CACHE-INV-NEXT:    s_endpgm
2774;
2775; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_acquire_cmpxchg:
2776; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2777; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2778; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2779; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2780; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2781; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2782; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2783; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2784;
2785; GFX90A-TGSPLIT-LABEL: local_singlethread_release_acquire_cmpxchg:
2786; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2787; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2788; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2789; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2790; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2791; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2792; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2793; GFX90A-TGSPLIT-NEXT:    s_endpgm
2794;
2795; GFX940-NOTTGSPLIT-LABEL: local_singlethread_release_acquire_cmpxchg:
2796; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2797; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2798; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2799; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2800; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2801; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2802; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2803; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2804;
2805; GFX940-TGSPLIT-LABEL: local_singlethread_release_acquire_cmpxchg:
2806; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2807; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2808; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2809; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2810; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2811; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2812; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2813; GFX940-TGSPLIT-NEXT:    s_endpgm
2814;
2815; GFX11-WGP-LABEL: local_singlethread_release_acquire_cmpxchg:
2816; GFX11-WGP:       ; %bb.0: ; %entry
2817; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2818; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2819; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2820; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
2821; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2822; GFX11-WGP-NEXT:    s_endpgm
2823;
2824; GFX11-CU-LABEL: local_singlethread_release_acquire_cmpxchg:
2825; GFX11-CU:       ; %bb.0: ; %entry
2826; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2827; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2828; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2829; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2830; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2831; GFX11-CU-NEXT:    s_endpgm
2832    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2833entry:
2834  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2835  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
2836  ret void
2837}
2838
2839define amdgpu_kernel void @local_singlethread_acq_rel_acquire_cmpxchg(
2840; GFX6-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2841; GFX6:       ; %bb.0: ; %entry
2842; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2843; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2844; GFX6-NEXT:    s_mov_b32 m0, -1
2845; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2846; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2847; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2848; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2849; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2850; GFX6-NEXT:    s_endpgm
2851;
2852; GFX7-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2853; GFX7:       ; %bb.0: ; %entry
2854; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2855; GFX7-NEXT:    s_mov_b32 m0, -1
2856; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2857; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2858; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2859; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2860; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2861; GFX7-NEXT:    s_endpgm
2862;
2863; GFX10-WGP-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2864; GFX10-WGP:       ; %bb.0: ; %entry
2865; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2866; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2867; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2868; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2869; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2870; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2871; GFX10-WGP-NEXT:    s_endpgm
2872;
2873; GFX10-CU-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2874; GFX10-CU:       ; %bb.0: ; %entry
2875; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2876; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2877; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2878; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2879; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2880; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2881; GFX10-CU-NEXT:    s_endpgm
2882;
2883; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2884; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2885; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2886; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2887; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2888; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2889; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2890; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2891; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2892; SKIP-CACHE-INV-NEXT:    s_endpgm
2893;
2894; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2895; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2896; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2897; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2898; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2899; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2900; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2901; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2902; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2903;
2904; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2905; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2906; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2907; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2908; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2909; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2910; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2911; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2912; GFX90A-TGSPLIT-NEXT:    s_endpgm
2913;
2914; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2915; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2916; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2917; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2918; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2919; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2920; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2921; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2922; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2923;
2924; GFX940-TGSPLIT-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2925; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2926; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2927; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2928; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2929; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2930; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2931; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2932; GFX940-TGSPLIT-NEXT:    s_endpgm
2933;
2934; GFX11-WGP-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2935; GFX11-WGP:       ; %bb.0: ; %entry
2936; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2937; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2938; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2939; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
2940; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2941; GFX11-WGP-NEXT:    s_endpgm
2942;
2943; GFX11-CU-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
2944; GFX11-CU:       ; %bb.0: ; %entry
2945; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
2946; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2947; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2948; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
2949; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
2950; GFX11-CU-NEXT:    s_endpgm
2951    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2952entry:
2953  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2954  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
2955  ret void
2956}
2957
2958define amdgpu_kernel void @local_singlethread_seq_cst_acquire_cmpxchg(
2959; GFX6-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
2960; GFX6:       ; %bb.0: ; %entry
2961; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2962; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2963; GFX6-NEXT:    s_mov_b32 m0, -1
2964; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2965; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2966; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2967; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2968; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2969; GFX6-NEXT:    s_endpgm
2970;
2971; GFX7-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
2972; GFX7:       ; %bb.0: ; %entry
2973; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2974; GFX7-NEXT:    s_mov_b32 m0, -1
2975; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2976; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2977; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2978; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2979; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2980; GFX7-NEXT:    s_endpgm
2981;
2982; GFX10-WGP-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
2983; GFX10-WGP:       ; %bb.0: ; %entry
2984; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2985; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2986; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2987; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2988; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2989; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2990; GFX10-WGP-NEXT:    s_endpgm
2991;
2992; GFX10-CU-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
2993; GFX10-CU:       ; %bb.0: ; %entry
2994; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2995; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2996; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2997; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2998; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2999; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3000; GFX10-CU-NEXT:    s_endpgm
3001;
3002; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
3003; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3004; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3005; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3006; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3007; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3008; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3009; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3010; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3011; SKIP-CACHE-INV-NEXT:    s_endpgm
3012;
3013; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
3014; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3015; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3016; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3017; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3018; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3019; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3020; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3021; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3022;
3023; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
3024; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3025; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3026; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3027; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3028; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3029; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3030; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3031; GFX90A-TGSPLIT-NEXT:    s_endpgm
3032;
3033; GFX940-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
3034; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3035; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3036; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3037; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3038; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3039; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3040; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3041; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3042;
3043; GFX940-TGSPLIT-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
3044; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3045; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3046; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3047; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3048; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3049; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3050; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3051; GFX940-TGSPLIT-NEXT:    s_endpgm
3052;
3053; GFX11-WGP-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
3054; GFX11-WGP:       ; %bb.0: ; %entry
3055; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3056; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3057; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3058; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
3059; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3060; GFX11-WGP-NEXT:    s_endpgm
3061;
3062; GFX11-CU-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
3063; GFX11-CU:       ; %bb.0: ; %entry
3064; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3065; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3066; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3067; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3068; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3069; GFX11-CU-NEXT:    s_endpgm
3070    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3071entry:
3072  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3073  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
3074  ret void
3075}
3076
3077define amdgpu_kernel void @local_singlethread_monotonic_seq_cst_cmpxchg(
3078; GFX6-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3079; GFX6:       ; %bb.0: ; %entry
3080; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3081; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3082; GFX6-NEXT:    s_mov_b32 m0, -1
3083; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3084; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3085; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3086; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3087; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3088; GFX6-NEXT:    s_endpgm
3089;
3090; GFX7-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3091; GFX7:       ; %bb.0: ; %entry
3092; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3093; GFX7-NEXT:    s_mov_b32 m0, -1
3094; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3095; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3096; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3097; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3098; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3099; GFX7-NEXT:    s_endpgm
3100;
3101; GFX10-WGP-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3102; GFX10-WGP:       ; %bb.0: ; %entry
3103; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3104; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3105; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3106; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3107; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3108; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3109; GFX10-WGP-NEXT:    s_endpgm
3110;
3111; GFX10-CU-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3112; GFX10-CU:       ; %bb.0: ; %entry
3113; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3114; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3115; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3116; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3117; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3118; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3119; GFX10-CU-NEXT:    s_endpgm
3120;
3121; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3122; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3123; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3124; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3125; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3126; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3127; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3128; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3129; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3130; SKIP-CACHE-INV-NEXT:    s_endpgm
3131;
3132; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3133; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3134; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3135; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3136; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3137; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3138; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3139; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3140; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3141;
3142; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3143; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3144; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3145; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3146; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3147; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3148; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3149; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3150; GFX90A-TGSPLIT-NEXT:    s_endpgm
3151;
3152; GFX940-NOTTGSPLIT-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3153; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3154; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3155; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3156; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3157; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3158; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3159; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3160; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3161;
3162; GFX940-TGSPLIT-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3163; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3164; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3165; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3166; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3167; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3168; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3169; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3170; GFX940-TGSPLIT-NEXT:    s_endpgm
3171;
3172; GFX11-WGP-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3173; GFX11-WGP:       ; %bb.0: ; %entry
3174; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3175; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3176; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3177; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
3178; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3179; GFX11-WGP-NEXT:    s_endpgm
3180;
3181; GFX11-CU-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
3182; GFX11-CU:       ; %bb.0: ; %entry
3183; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3184; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3185; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3186; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3187; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3188; GFX11-CU-NEXT:    s_endpgm
3189    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3190entry:
3191  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3192  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic seq_cst
3193  ret void
3194}
3195
3196define amdgpu_kernel void @local_singlethread_acquire_seq_cst_cmpxchg(
3197; GFX6-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3198; GFX6:       ; %bb.0: ; %entry
3199; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3200; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3201; GFX6-NEXT:    s_mov_b32 m0, -1
3202; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3203; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3204; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3205; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3206; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3207; GFX6-NEXT:    s_endpgm
3208;
3209; GFX7-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3210; GFX7:       ; %bb.0: ; %entry
3211; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3212; GFX7-NEXT:    s_mov_b32 m0, -1
3213; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3214; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3215; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3216; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3217; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3218; GFX7-NEXT:    s_endpgm
3219;
3220; GFX10-WGP-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3221; GFX10-WGP:       ; %bb.0: ; %entry
3222; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3223; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3224; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3225; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3226; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3227; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3228; GFX10-WGP-NEXT:    s_endpgm
3229;
3230; GFX10-CU-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3231; GFX10-CU:       ; %bb.0: ; %entry
3232; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3233; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3234; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3235; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3236; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3237; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3238; GFX10-CU-NEXT:    s_endpgm
3239;
3240; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3241; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3242; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3243; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3244; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3245; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3246; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3247; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3248; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3249; SKIP-CACHE-INV-NEXT:    s_endpgm
3250;
3251; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3252; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3253; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3254; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3255; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3256; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3257; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3258; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3259; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3260;
3261; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3262; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3263; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3264; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3265; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3266; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3267; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3268; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3269; GFX90A-TGSPLIT-NEXT:    s_endpgm
3270;
3271; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3272; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3273; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3274; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3275; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3276; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3277; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3278; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3279; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3280;
3281; GFX940-TGSPLIT-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3282; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3283; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3284; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3285; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3286; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3287; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3288; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3289; GFX940-TGSPLIT-NEXT:    s_endpgm
3290;
3291; GFX11-WGP-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3292; GFX11-WGP:       ; %bb.0: ; %entry
3293; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3294; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3295; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3296; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
3297; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3298; GFX11-WGP-NEXT:    s_endpgm
3299;
3300; GFX11-CU-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
3301; GFX11-CU:       ; %bb.0: ; %entry
3302; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3303; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3304; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3305; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3306; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3307; GFX11-CU-NEXT:    s_endpgm
3308    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3309entry:
3310  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3311  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire seq_cst
3312  ret void
3313}
3314
3315define amdgpu_kernel void @local_singlethread_release_seq_cst_cmpxchg(
3316; GFX6-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3317; GFX6:       ; %bb.0: ; %entry
3318; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3319; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3320; GFX6-NEXT:    s_mov_b32 m0, -1
3321; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3322; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3323; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3324; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3325; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3326; GFX6-NEXT:    s_endpgm
3327;
3328; GFX7-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3329; GFX7:       ; %bb.0: ; %entry
3330; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3331; GFX7-NEXT:    s_mov_b32 m0, -1
3332; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3333; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3334; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3335; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3336; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3337; GFX7-NEXT:    s_endpgm
3338;
3339; GFX10-WGP-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3340; GFX10-WGP:       ; %bb.0: ; %entry
3341; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3342; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3343; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3344; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3345; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3346; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3347; GFX10-WGP-NEXT:    s_endpgm
3348;
3349; GFX10-CU-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3350; GFX10-CU:       ; %bb.0: ; %entry
3351; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3352; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3353; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3354; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3355; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3356; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3357; GFX10-CU-NEXT:    s_endpgm
3358;
3359; SKIP-CACHE-INV-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3360; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3361; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3362; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3363; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3364; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3365; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3366; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3367; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3368; SKIP-CACHE-INV-NEXT:    s_endpgm
3369;
3370; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3371; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3372; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3373; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3374; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3375; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3376; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3377; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3378; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3379;
3380; GFX90A-TGSPLIT-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3381; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3382; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3383; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3384; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3385; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3386; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3387; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3388; GFX90A-TGSPLIT-NEXT:    s_endpgm
3389;
3390; GFX940-NOTTGSPLIT-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3391; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3392; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3393; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3394; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3395; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3396; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3397; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3398; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3399;
3400; GFX940-TGSPLIT-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3401; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3402; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3403; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3404; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3405; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3406; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3407; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3408; GFX940-TGSPLIT-NEXT:    s_endpgm
3409;
3410; GFX11-WGP-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3411; GFX11-WGP:       ; %bb.0: ; %entry
3412; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3413; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3414; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3415; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
3416; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3417; GFX11-WGP-NEXT:    s_endpgm
3418;
3419; GFX11-CU-LABEL: local_singlethread_release_seq_cst_cmpxchg:
3420; GFX11-CU:       ; %bb.0: ; %entry
3421; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3422; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3423; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3424; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3425; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3426; GFX11-CU-NEXT:    s_endpgm
3427    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3428entry:
3429  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3430  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release seq_cst
3431  ret void
3432}
3433
3434define amdgpu_kernel void @local_singlethread_acq_rel_seq_cst_cmpxchg(
3435; GFX6-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3436; GFX6:       ; %bb.0: ; %entry
3437; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3438; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3439; GFX6-NEXT:    s_mov_b32 m0, -1
3440; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3441; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3442; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3443; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3444; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3445; GFX6-NEXT:    s_endpgm
3446;
3447; GFX7-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3448; GFX7:       ; %bb.0: ; %entry
3449; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3450; GFX7-NEXT:    s_mov_b32 m0, -1
3451; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3452; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3453; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3454; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3455; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3456; GFX7-NEXT:    s_endpgm
3457;
3458; GFX10-WGP-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3459; GFX10-WGP:       ; %bb.0: ; %entry
3460; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3461; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3462; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3463; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3464; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3465; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3466; GFX10-WGP-NEXT:    s_endpgm
3467;
3468; GFX10-CU-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3469; GFX10-CU:       ; %bb.0: ; %entry
3470; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3471; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3472; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3473; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3474; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3475; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3476; GFX10-CU-NEXT:    s_endpgm
3477;
3478; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3479; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3480; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3481; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3482; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3483; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3484; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3485; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3486; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3487; SKIP-CACHE-INV-NEXT:    s_endpgm
3488;
3489; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3490; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3491; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3492; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3493; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3494; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3495; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3496; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3497; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3498;
3499; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3500; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3501; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3502; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3503; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3504; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3505; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3506; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3507; GFX90A-TGSPLIT-NEXT:    s_endpgm
3508;
3509; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3510; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3511; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3512; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3513; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3514; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3515; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3516; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3517; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3518;
3519; GFX940-TGSPLIT-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3520; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3521; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3522; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3523; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3524; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3525; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3526; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3527; GFX940-TGSPLIT-NEXT:    s_endpgm
3528;
3529; GFX11-WGP-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3530; GFX11-WGP:       ; %bb.0: ; %entry
3531; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3532; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3533; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3534; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
3535; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3536; GFX11-WGP-NEXT:    s_endpgm
3537;
3538; GFX11-CU-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
3539; GFX11-CU:       ; %bb.0: ; %entry
3540; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3541; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3542; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3543; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3544; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3545; GFX11-CU-NEXT:    s_endpgm
3546    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3547entry:
3548  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3549  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel seq_cst
3550  ret void
3551}
3552
3553define amdgpu_kernel void @local_singlethread_seq_cst_seq_cst_cmpxchg(
3554; GFX6-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3555; GFX6:       ; %bb.0: ; %entry
3556; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3557; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3558; GFX6-NEXT:    s_mov_b32 m0, -1
3559; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3560; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3561; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3562; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3563; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3564; GFX6-NEXT:    s_endpgm
3565;
3566; GFX7-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3567; GFX7:       ; %bb.0: ; %entry
3568; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3569; GFX7-NEXT:    s_mov_b32 m0, -1
3570; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3571; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3572; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3573; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3574; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3575; GFX7-NEXT:    s_endpgm
3576;
3577; GFX10-WGP-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3578; GFX10-WGP:       ; %bb.0: ; %entry
3579; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3580; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3581; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3582; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3583; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3584; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3585; GFX10-WGP-NEXT:    s_endpgm
3586;
3587; GFX10-CU-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3588; GFX10-CU:       ; %bb.0: ; %entry
3589; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3590; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3591; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3592; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3593; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3594; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3595; GFX10-CU-NEXT:    s_endpgm
3596;
3597; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3598; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3599; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3600; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3601; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3602; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3603; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3604; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3605; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3606; SKIP-CACHE-INV-NEXT:    s_endpgm
3607;
3608; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3609; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3610; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3611; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3612; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3613; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3614; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3615; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3616; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3617;
3618; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3619; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3620; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3621; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3622; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3623; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3624; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3625; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3626; GFX90A-TGSPLIT-NEXT:    s_endpgm
3627;
3628; GFX940-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3629; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3630; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3631; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3632; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3633; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3634; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3635; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3636; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3637;
3638; GFX940-TGSPLIT-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3639; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3640; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3641; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3642; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3643; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3644; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3645; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
3646; GFX940-TGSPLIT-NEXT:    s_endpgm
3647;
3648; GFX11-WGP-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3649; GFX11-WGP:       ; %bb.0: ; %entry
3650; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3651; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3652; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3653; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
3654; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3655; GFX11-WGP-NEXT:    s_endpgm
3656;
3657; GFX11-CU-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
3658; GFX11-CU:       ; %bb.0: ; %entry
3659; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3660; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3661; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3662; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3663; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
3664; GFX11-CU-NEXT:    s_endpgm
3665    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3666entry:
3667  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3668  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
3669  ret void
3670}
3671
3672define amdgpu_kernel void @local_singlethread_monotonic_monotonic_ret_cmpxchg(
3673; GFX6-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3674; GFX6:       ; %bb.0: ; %entry
3675; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3676; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3677; GFX6-NEXT:    s_mov_b32 m0, -1
3678; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3679; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3680; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3681; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3682; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3683; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3684; GFX6-NEXT:    ds_write_b32 v0, v1
3685; GFX6-NEXT:    s_endpgm
3686;
3687; GFX7-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3688; GFX7:       ; %bb.0: ; %entry
3689; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3690; GFX7-NEXT:    s_mov_b32 m0, -1
3691; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3692; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3693; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3694; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3695; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3696; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3697; GFX7-NEXT:    ds_write_b32 v0, v1
3698; GFX7-NEXT:    s_endpgm
3699;
3700; GFX10-WGP-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3701; GFX10-WGP:       ; %bb.0: ; %entry
3702; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3703; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3704; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3705; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3706; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3707; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3708; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3709; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
3710; GFX10-WGP-NEXT:    s_endpgm
3711;
3712; GFX10-CU-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3713; GFX10-CU:       ; %bb.0: ; %entry
3714; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3715; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3716; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3717; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3718; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3719; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3720; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3721; GFX10-CU-NEXT:    ds_write_b32 v0, v1
3722; GFX10-CU-NEXT:    s_endpgm
3723;
3724; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3725; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3726; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3727; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3728; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3729; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3730; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3731; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3732; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3733; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3734; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
3735; SKIP-CACHE-INV-NEXT:    s_endpgm
3736;
3737; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3738; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3739; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3740; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3741; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3742; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3743; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3744; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3745; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3746; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
3747; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3748;
3749; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3750; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3751; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3752; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3753; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3754; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3755; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3756; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3757; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3758; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
3759; GFX90A-TGSPLIT-NEXT:    s_endpgm
3760;
3761; GFX940-NOTTGSPLIT-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3762; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3763; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3764; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3765; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3766; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3767; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3768; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3769; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3770; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
3771; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3772;
3773; GFX940-TGSPLIT-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3774; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3775; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3776; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3777; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3778; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3779; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3780; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3781; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3782; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
3783; GFX940-TGSPLIT-NEXT:    s_endpgm
3784;
3785; GFX11-WGP-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3786; GFX11-WGP:       ; %bb.0: ; %entry
3787; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3788; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3789; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3790; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
3791; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
3792; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3793; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
3794; GFX11-WGP-NEXT:    s_endpgm
3795;
3796; GFX11-CU-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
3797; GFX11-CU:       ; %bb.0: ; %entry
3798; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3799; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3800; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3801; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3802; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
3803; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3804; GFX11-CU-NEXT:    ds_store_b32 v0, v1
3805; GFX11-CU-NEXT:    s_endpgm
3806    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3807entry:
3808  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3809  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
3810  %val0 = extractvalue { i32, i1 } %val, 0
3811  store i32 %val0, i32 addrspace(3)* %out, align 4
3812  ret void
3813}
3814
3815define amdgpu_kernel void @local_singlethread_acquire_monotonic_ret_cmpxchg(
3816; GFX6-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3817; GFX6:       ; %bb.0: ; %entry
3818; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3819; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3820; GFX6-NEXT:    s_mov_b32 m0, -1
3821; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3822; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3823; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3824; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3825; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3826; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3827; GFX6-NEXT:    ds_write_b32 v0, v1
3828; GFX6-NEXT:    s_endpgm
3829;
3830; GFX7-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3831; GFX7:       ; %bb.0: ; %entry
3832; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3833; GFX7-NEXT:    s_mov_b32 m0, -1
3834; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3835; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3836; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3837; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3838; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3839; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3840; GFX7-NEXT:    ds_write_b32 v0, v1
3841; GFX7-NEXT:    s_endpgm
3842;
3843; GFX10-WGP-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3844; GFX10-WGP:       ; %bb.0: ; %entry
3845; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3846; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3847; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3848; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3849; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3850; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3851; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3852; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
3853; GFX10-WGP-NEXT:    s_endpgm
3854;
3855; GFX10-CU-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3856; GFX10-CU:       ; %bb.0: ; %entry
3857; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3858; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3859; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3860; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3861; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3862; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3863; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3864; GFX10-CU-NEXT:    ds_write_b32 v0, v1
3865; GFX10-CU-NEXT:    s_endpgm
3866;
3867; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3868; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3869; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3870; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3871; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3872; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3873; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3874; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3875; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3876; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3877; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
3878; SKIP-CACHE-INV-NEXT:    s_endpgm
3879;
3880; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3881; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3882; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3883; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3884; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3885; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3886; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3887; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3888; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3889; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
3890; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3891;
3892; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3893; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3894; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3895; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3896; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3897; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3898; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3899; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3900; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3901; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
3902; GFX90A-TGSPLIT-NEXT:    s_endpgm
3903;
3904; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3905; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3906; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3907; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3908; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3909; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3910; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3911; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3912; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3913; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
3914; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3915;
3916; GFX940-TGSPLIT-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3917; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3918; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3919; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3920; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3921; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3922; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3923; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3924; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3925; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
3926; GFX940-TGSPLIT-NEXT:    s_endpgm
3927;
3928; GFX11-WGP-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3929; GFX11-WGP:       ; %bb.0: ; %entry
3930; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3931; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3932; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3933; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
3934; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
3935; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3936; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
3937; GFX11-WGP-NEXT:    s_endpgm
3938;
3939; GFX11-CU-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
3940; GFX11-CU:       ; %bb.0: ; %entry
3941; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
3942; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3943; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3944; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
3945; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
3946; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3947; GFX11-CU-NEXT:    ds_store_b32 v0, v1
3948; GFX11-CU-NEXT:    s_endpgm
3949    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3950entry:
3951  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3952  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
3953  %val0 = extractvalue { i32, i1 } %val, 0
3954  store i32 %val0, i32 addrspace(3)* %out, align 4
3955  ret void
3956}
3957
3958define amdgpu_kernel void @local_singlethread_release_monotonic_ret_cmpxchg(
3959; GFX6-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
3960; GFX6:       ; %bb.0: ; %entry
3961; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3962; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3963; GFX6-NEXT:    s_mov_b32 m0, -1
3964; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3965; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3966; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3967; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3968; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3969; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3970; GFX6-NEXT:    ds_write_b32 v0, v1
3971; GFX6-NEXT:    s_endpgm
3972;
3973; GFX7-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
3974; GFX7:       ; %bb.0: ; %entry
3975; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3976; GFX7-NEXT:    s_mov_b32 m0, -1
3977; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3978; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3979; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3980; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3981; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3982; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3983; GFX7-NEXT:    ds_write_b32 v0, v1
3984; GFX7-NEXT:    s_endpgm
3985;
3986; GFX10-WGP-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
3987; GFX10-WGP:       ; %bb.0: ; %entry
3988; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3989; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3990; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3991; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3992; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3993; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3994; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3995; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
3996; GFX10-WGP-NEXT:    s_endpgm
3997;
3998; GFX10-CU-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
3999; GFX10-CU:       ; %bb.0: ; %entry
4000; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4001; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4002; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4003; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4004; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4005; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4006; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4007; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4008; GFX10-CU-NEXT:    s_endpgm
4009;
4010; SKIP-CACHE-INV-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
4011; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4012; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4013; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4014; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4015; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4016; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
4017; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
4018; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4019; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4020; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
4021; SKIP-CACHE-INV-NEXT:    s_endpgm
4022;
4023; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
4024; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4025; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4026; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4027; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4028; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4029; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4030; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4031; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4032; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4033; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4034;
4035; GFX90A-TGSPLIT-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
4036; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4037; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4038; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4039; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4040; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4041; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4042; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4043; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4044; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4045; GFX90A-TGSPLIT-NEXT:    s_endpgm
4046;
4047; GFX940-NOTTGSPLIT-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
4048; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4049; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4050; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4051; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4052; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4053; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4054; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4055; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4056; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4057; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4058;
4059; GFX940-TGSPLIT-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
4060; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4061; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4062; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4063; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4064; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4065; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4066; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4067; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4068; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4069; GFX940-TGSPLIT-NEXT:    s_endpgm
4070;
4071; GFX11-WGP-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
4072; GFX11-WGP:       ; %bb.0: ; %entry
4073; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4074; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4075; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4076; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
4077; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4078; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4079; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
4080; GFX11-WGP-NEXT:    s_endpgm
4081;
4082; GFX11-CU-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
4083; GFX11-CU:       ; %bb.0: ; %entry
4084; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4085; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4086; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4087; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4088; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4089; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4090; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4091; GFX11-CU-NEXT:    s_endpgm
4092    i32 addrspace(3)* %out, i32 %in, i32 %old) {
4093entry:
4094  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
4095  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
4096  %val0 = extractvalue { i32, i1 } %val, 0
4097  store i32 %val0, i32 addrspace(3)* %out, align 4
4098  ret void
4099}
4100
4101define amdgpu_kernel void @local_singlethread_acq_rel_monotonic_ret_cmpxchg(
4102; GFX6-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4103; GFX6:       ; %bb.0: ; %entry
4104; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
4105; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
4106; GFX6-NEXT:    s_mov_b32 m0, -1
4107; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4108; GFX6-NEXT:    v_mov_b32_e32 v0, s2
4109; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4110; GFX6-NEXT:    v_mov_b32_e32 v2, s0
4111; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4112; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4113; GFX6-NEXT:    ds_write_b32 v0, v1
4114; GFX6-NEXT:    s_endpgm
4115;
4116; GFX7-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4117; GFX7:       ; %bb.0: ; %entry
4118; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4119; GFX7-NEXT:    s_mov_b32 m0, -1
4120; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4121; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4122; GFX7-NEXT:    v_mov_b32_e32 v1, s2
4123; GFX7-NEXT:    v_mov_b32_e32 v2, s1
4124; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4125; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4126; GFX7-NEXT:    ds_write_b32 v0, v1
4127; GFX7-NEXT:    s_endpgm
4128;
4129; GFX10-WGP-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4130; GFX10-WGP:       ; %bb.0: ; %entry
4131; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4132; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4133; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4134; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4135; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
4136; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4137; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4138; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
4139; GFX10-WGP-NEXT:    s_endpgm
4140;
4141; GFX10-CU-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4142; GFX10-CU:       ; %bb.0: ; %entry
4143; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4144; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4145; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4146; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4147; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4148; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4149; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4150; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4151; GFX10-CU-NEXT:    s_endpgm
4152;
4153; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4154; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4155; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4156; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4157; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4158; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4159; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
4160; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
4161; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4162; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4163; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
4164; SKIP-CACHE-INV-NEXT:    s_endpgm
4165;
4166; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4167; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4168; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4169; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4170; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4171; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4172; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4173; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4174; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4175; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4176; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4177;
4178; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4179; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4180; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4181; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4182; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4183; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4184; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4185; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4186; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4187; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4188; GFX90A-TGSPLIT-NEXT:    s_endpgm
4189;
4190; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4191; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4192; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4193; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4194; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4195; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4196; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4197; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4198; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4199; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4200; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4201;
4202; GFX940-TGSPLIT-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4203; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4204; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4205; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4206; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4207; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4208; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4209; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4210; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4211; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4212; GFX940-TGSPLIT-NEXT:    s_endpgm
4213;
4214; GFX11-WGP-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4215; GFX11-WGP:       ; %bb.0: ; %entry
4216; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4217; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4218; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4219; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
4220; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4221; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4222; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
4223; GFX11-WGP-NEXT:    s_endpgm
4224;
4225; GFX11-CU-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
4226; GFX11-CU:       ; %bb.0: ; %entry
4227; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4228; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4229; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4230; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4231; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4232; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4233; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4234; GFX11-CU-NEXT:    s_endpgm
4235    i32 addrspace(3)* %out, i32 %in, i32 %old) {
4236entry:
4237  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
4238  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
4239  %val0 = extractvalue { i32, i1 } %val, 0
4240  store i32 %val0, i32 addrspace(3)* %out, align 4
4241  ret void
4242}
4243
4244define amdgpu_kernel void @local_singlethread_seq_cst_monotonic_ret_cmpxchg(
4245; GFX6-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4246; GFX6:       ; %bb.0: ; %entry
4247; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
4248; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
4249; GFX6-NEXT:    s_mov_b32 m0, -1
4250; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4251; GFX6-NEXT:    v_mov_b32_e32 v0, s2
4252; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4253; GFX6-NEXT:    v_mov_b32_e32 v2, s0
4254; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4255; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4256; GFX6-NEXT:    ds_write_b32 v0, v1
4257; GFX6-NEXT:    s_endpgm
4258;
4259; GFX7-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4260; GFX7:       ; %bb.0: ; %entry
4261; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4262; GFX7-NEXT:    s_mov_b32 m0, -1
4263; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4264; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4265; GFX7-NEXT:    v_mov_b32_e32 v1, s2
4266; GFX7-NEXT:    v_mov_b32_e32 v2, s1
4267; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4268; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4269; GFX7-NEXT:    ds_write_b32 v0, v1
4270; GFX7-NEXT:    s_endpgm
4271;
4272; GFX10-WGP-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4273; GFX10-WGP:       ; %bb.0: ; %entry
4274; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4275; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4276; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4277; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4278; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
4279; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4280; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4281; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
4282; GFX10-WGP-NEXT:    s_endpgm
4283;
4284; GFX10-CU-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4285; GFX10-CU:       ; %bb.0: ; %entry
4286; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4287; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4288; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4289; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4290; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4291; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4292; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4293; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4294; GFX10-CU-NEXT:    s_endpgm
4295;
4296; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4297; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4298; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4299; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4300; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4301; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4302; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
4303; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
4304; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4305; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4306; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
4307; SKIP-CACHE-INV-NEXT:    s_endpgm
4308;
4309; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4310; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4311; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4312; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4313; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4314; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4315; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4316; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4317; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4318; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4319; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4320;
4321; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4322; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4323; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4324; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4325; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4326; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4327; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4328; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4329; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4330; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4331; GFX90A-TGSPLIT-NEXT:    s_endpgm
4332;
4333; GFX940-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4334; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4335; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4336; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4337; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4338; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4339; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4340; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4341; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4342; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4343; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4344;
4345; GFX940-TGSPLIT-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4346; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4347; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4348; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4349; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4350; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4351; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4352; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4353; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4354; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4355; GFX940-TGSPLIT-NEXT:    s_endpgm
4356;
4357; GFX11-WGP-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4358; GFX11-WGP:       ; %bb.0: ; %entry
4359; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4360; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4361; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4362; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
4363; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4364; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4365; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
4366; GFX11-WGP-NEXT:    s_endpgm
4367;
4368; GFX11-CU-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
4369; GFX11-CU:       ; %bb.0: ; %entry
4370; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4371; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4372; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4373; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4374; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4375; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4376; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4377; GFX11-CU-NEXT:    s_endpgm
4378    i32 addrspace(3)* %out, i32 %in, i32 %old) {
4379entry:
4380  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
4381  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
4382  %val0 = extractvalue { i32, i1 } %val, 0
4383  store i32 %val0, i32 addrspace(3)* %out, align 4
4384  ret void
4385}
4386
4387define amdgpu_kernel void @local_singlethread_monotonic_acquire_ret_cmpxchg(
4388; GFX6-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4389; GFX6:       ; %bb.0: ; %entry
4390; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
4391; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
4392; GFX6-NEXT:    s_mov_b32 m0, -1
4393; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4394; GFX6-NEXT:    v_mov_b32_e32 v0, s2
4395; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4396; GFX6-NEXT:    v_mov_b32_e32 v2, s0
4397; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4398; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4399; GFX6-NEXT:    ds_write_b32 v0, v1
4400; GFX6-NEXT:    s_endpgm
4401;
4402; GFX7-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4403; GFX7:       ; %bb.0: ; %entry
4404; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4405; GFX7-NEXT:    s_mov_b32 m0, -1
4406; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4407; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4408; GFX7-NEXT:    v_mov_b32_e32 v1, s2
4409; GFX7-NEXT:    v_mov_b32_e32 v2, s1
4410; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4411; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4412; GFX7-NEXT:    ds_write_b32 v0, v1
4413; GFX7-NEXT:    s_endpgm
4414;
4415; GFX10-WGP-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4416; GFX10-WGP:       ; %bb.0: ; %entry
4417; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4418; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4419; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4420; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4421; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
4422; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4423; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4424; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
4425; GFX10-WGP-NEXT:    s_endpgm
4426;
4427; GFX10-CU-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4428; GFX10-CU:       ; %bb.0: ; %entry
4429; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4430; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4431; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4432; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4433; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4434; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4435; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4436; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4437; GFX10-CU-NEXT:    s_endpgm
4438;
4439; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4440; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4441; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4442; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4443; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4444; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4445; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
4446; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
4447; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4448; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4449; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
4450; SKIP-CACHE-INV-NEXT:    s_endpgm
4451;
4452; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4453; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4454; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4455; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4456; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4457; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4458; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4459; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4460; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4461; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4462; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4463;
4464; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4465; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4466; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4467; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4468; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4469; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4470; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4471; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4472; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4473; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4474; GFX90A-TGSPLIT-NEXT:    s_endpgm
4475;
4476; GFX940-NOTTGSPLIT-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4477; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4478; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4479; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4480; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4481; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4482; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4483; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4484; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4485; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4486; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4487;
4488; GFX940-TGSPLIT-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4489; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4490; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4491; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4492; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4493; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4494; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4495; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4496; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4497; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4498; GFX940-TGSPLIT-NEXT:    s_endpgm
4499;
4500; GFX11-WGP-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4501; GFX11-WGP:       ; %bb.0: ; %entry
4502; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4503; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4504; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4505; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
4506; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4507; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4508; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
4509; GFX11-WGP-NEXT:    s_endpgm
4510;
4511; GFX11-CU-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
4512; GFX11-CU:       ; %bb.0: ; %entry
4513; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4514; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4515; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4516; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4517; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4518; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4519; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4520; GFX11-CU-NEXT:    s_endpgm
4521    i32 addrspace(3)* %out, i32 %in, i32 %old) {
4522entry:
4523  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
4524  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic acquire
4525  %val0 = extractvalue { i32, i1 } %val, 0
4526  store i32 %val0, i32 addrspace(3)* %out, align 4
4527  ret void
4528}
4529
4530define amdgpu_kernel void @local_singlethread_acquire_acquire_ret_cmpxchg(
4531; GFX6-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4532; GFX6:       ; %bb.0: ; %entry
4533; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
4534; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
4535; GFX6-NEXT:    s_mov_b32 m0, -1
4536; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4537; GFX6-NEXT:    v_mov_b32_e32 v0, s2
4538; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4539; GFX6-NEXT:    v_mov_b32_e32 v2, s0
4540; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4541; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4542; GFX6-NEXT:    ds_write_b32 v0, v1
4543; GFX6-NEXT:    s_endpgm
4544;
4545; GFX7-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4546; GFX7:       ; %bb.0: ; %entry
4547; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4548; GFX7-NEXT:    s_mov_b32 m0, -1
4549; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4550; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4551; GFX7-NEXT:    v_mov_b32_e32 v1, s2
4552; GFX7-NEXT:    v_mov_b32_e32 v2, s1
4553; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4554; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4555; GFX7-NEXT:    ds_write_b32 v0, v1
4556; GFX7-NEXT:    s_endpgm
4557;
4558; GFX10-WGP-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4559; GFX10-WGP:       ; %bb.0: ; %entry
4560; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4561; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4562; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4563; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4564; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
4565; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4566; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4567; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
4568; GFX10-WGP-NEXT:    s_endpgm
4569;
4570; GFX10-CU-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4571; GFX10-CU:       ; %bb.0: ; %entry
4572; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4573; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4574; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4575; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4576; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4577; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4578; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4579; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4580; GFX10-CU-NEXT:    s_endpgm
4581;
4582; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4583; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4584; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4585; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4586; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4587; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4588; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
4589; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
4590; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4591; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4592; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
4593; SKIP-CACHE-INV-NEXT:    s_endpgm
4594;
4595; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4596; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4597; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4598; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4599; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4600; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4601; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4602; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4603; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4604; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4605; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4606;
4607; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4608; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4609; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4610; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4611; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4612; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4613; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4614; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4615; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4616; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4617; GFX90A-TGSPLIT-NEXT:    s_endpgm
4618;
4619; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4620; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4621; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4622; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4623; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4624; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4625; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4626; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4627; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4628; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4629; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4630;
4631; GFX940-TGSPLIT-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4632; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4633; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4634; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4635; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4636; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4637; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4638; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4639; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4640; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4641; GFX940-TGSPLIT-NEXT:    s_endpgm
4642;
4643; GFX11-WGP-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4644; GFX11-WGP:       ; %bb.0: ; %entry
4645; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4646; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4647; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4648; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
4649; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4650; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4651; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
4652; GFX11-WGP-NEXT:    s_endpgm
4653;
4654; GFX11-CU-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
4655; GFX11-CU:       ; %bb.0: ; %entry
4656; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4657; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4658; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4659; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4660; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4661; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4662; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4663; GFX11-CU-NEXT:    s_endpgm
4664    i32 addrspace(3)* %out, i32 %in, i32 %old) {
4665entry:
4666  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
4667  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
4668  %val0 = extractvalue { i32, i1 } %val, 0
4669  store i32 %val0, i32 addrspace(3)* %out, align 4
4670  ret void
4671}
4672
4673define amdgpu_kernel void @local_singlethread_release_acquire_ret_cmpxchg(
4674; GFX6-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4675; GFX6:       ; %bb.0: ; %entry
4676; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
4677; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
4678; GFX6-NEXT:    s_mov_b32 m0, -1
4679; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4680; GFX6-NEXT:    v_mov_b32_e32 v0, s2
4681; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4682; GFX6-NEXT:    v_mov_b32_e32 v2, s0
4683; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4684; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4685; GFX6-NEXT:    ds_write_b32 v0, v1
4686; GFX6-NEXT:    s_endpgm
4687;
4688; GFX7-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4689; GFX7:       ; %bb.0: ; %entry
4690; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4691; GFX7-NEXT:    s_mov_b32 m0, -1
4692; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4693; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4694; GFX7-NEXT:    v_mov_b32_e32 v1, s2
4695; GFX7-NEXT:    v_mov_b32_e32 v2, s1
4696; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4697; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4698; GFX7-NEXT:    ds_write_b32 v0, v1
4699; GFX7-NEXT:    s_endpgm
4700;
4701; GFX10-WGP-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4702; GFX10-WGP:       ; %bb.0: ; %entry
4703; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4704; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4705; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4706; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4707; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
4708; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4709; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4710; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
4711; GFX10-WGP-NEXT:    s_endpgm
4712;
4713; GFX10-CU-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4714; GFX10-CU:       ; %bb.0: ; %entry
4715; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4716; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4717; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4718; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4719; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4720; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4721; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4722; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4723; GFX10-CU-NEXT:    s_endpgm
4724;
4725; SKIP-CACHE-INV-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4726; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4727; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4728; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4729; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4730; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4731; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
4732; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
4733; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4734; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4735; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
4736; SKIP-CACHE-INV-NEXT:    s_endpgm
4737;
4738; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4739; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4740; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4741; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4742; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4743; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4744; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4745; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4746; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4747; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4748; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4749;
4750; GFX90A-TGSPLIT-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4751; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4752; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4753; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4754; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4755; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4756; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4757; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4758; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4759; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4760; GFX90A-TGSPLIT-NEXT:    s_endpgm
4761;
4762; GFX940-NOTTGSPLIT-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4763; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4764; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4765; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4766; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4767; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4768; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4769; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4770; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4771; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4772; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4773;
4774; GFX940-TGSPLIT-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4775; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4776; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4777; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4778; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4779; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4780; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4781; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4782; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4783; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4784; GFX940-TGSPLIT-NEXT:    s_endpgm
4785;
4786; GFX11-WGP-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4787; GFX11-WGP:       ; %bb.0: ; %entry
4788; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4789; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4790; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4791; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
4792; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4793; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4794; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
4795; GFX11-WGP-NEXT:    s_endpgm
4796;
4797; GFX11-CU-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
4798; GFX11-CU:       ; %bb.0: ; %entry
4799; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4800; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4801; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4802; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4803; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4804; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4805; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4806; GFX11-CU-NEXT:    s_endpgm
4807    i32 addrspace(3)* %out, i32 %in, i32 %old) {
4808entry:
4809  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
4810  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
4811  %val0 = extractvalue { i32, i1 } %val, 0
4812  store i32 %val0, i32 addrspace(3)* %out, align 4
4813  ret void
4814}
4815
4816define amdgpu_kernel void @local_singlethread_acq_rel_acquire_ret_cmpxchg(
4817; GFX6-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4818; GFX6:       ; %bb.0: ; %entry
4819; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
4820; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
4821; GFX6-NEXT:    s_mov_b32 m0, -1
4822; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4823; GFX6-NEXT:    v_mov_b32_e32 v0, s2
4824; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4825; GFX6-NEXT:    v_mov_b32_e32 v2, s0
4826; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4827; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4828; GFX6-NEXT:    ds_write_b32 v0, v1
4829; GFX6-NEXT:    s_endpgm
4830;
4831; GFX7-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4832; GFX7:       ; %bb.0: ; %entry
4833; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4834; GFX7-NEXT:    s_mov_b32 m0, -1
4835; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4836; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4837; GFX7-NEXT:    v_mov_b32_e32 v1, s2
4838; GFX7-NEXT:    v_mov_b32_e32 v2, s1
4839; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4840; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4841; GFX7-NEXT:    ds_write_b32 v0, v1
4842; GFX7-NEXT:    s_endpgm
4843;
4844; GFX10-WGP-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4845; GFX10-WGP:       ; %bb.0: ; %entry
4846; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4847; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4848; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4849; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4850; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
4851; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4852; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4853; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
4854; GFX10-WGP-NEXT:    s_endpgm
4855;
4856; GFX10-CU-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4857; GFX10-CU:       ; %bb.0: ; %entry
4858; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4859; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4860; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4861; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4862; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
4863; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4864; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4865; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4866; GFX10-CU-NEXT:    s_endpgm
4867;
4868; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4869; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4870; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4871; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4872; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4873; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4874; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
4875; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
4876; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4877; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4878; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
4879; SKIP-CACHE-INV-NEXT:    s_endpgm
4880;
4881; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4882; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4883; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4884; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4885; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4886; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4887; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4888; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4889; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4890; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4891; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4892;
4893; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4894; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4895; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4896; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4897; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4898; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4899; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4900; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4901; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4902; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4903; GFX90A-TGSPLIT-NEXT:    s_endpgm
4904;
4905; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4906; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4907; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4908; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4909; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4910; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4911; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4912; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4913; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4914; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4915; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4916;
4917; GFX940-TGSPLIT-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4918; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4919; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4920; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4921; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4922; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4923; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
4924; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4925; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4926; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4927; GFX940-TGSPLIT-NEXT:    s_endpgm
4928;
4929; GFX11-WGP-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4930; GFX11-WGP:       ; %bb.0: ; %entry
4931; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4932; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4933; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4934; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
4935; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4936; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4937; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
4938; GFX11-WGP-NEXT:    s_endpgm
4939;
4940; GFX11-CU-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
4941; GFX11-CU:       ; %bb.0: ; %entry
4942; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
4943; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4944; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4945; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
4946; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
4947; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4948; GFX11-CU-NEXT:    ds_store_b32 v0, v1
4949; GFX11-CU-NEXT:    s_endpgm
4950    i32 addrspace(3)* %out, i32 %in, i32 %old) {
4951entry:
4952  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
4953  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
4954  %val0 = extractvalue { i32, i1 } %val, 0
4955  store i32 %val0, i32 addrspace(3)* %out, align 4
4956  ret void
4957}
4958
4959define amdgpu_kernel void @local_singlethread_seq_cst_acquire_ret_cmpxchg(
4960; GFX6-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
4961; GFX6:       ; %bb.0: ; %entry
4962; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
4963; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
4964; GFX6-NEXT:    s_mov_b32 m0, -1
4965; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4966; GFX6-NEXT:    v_mov_b32_e32 v0, s2
4967; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4968; GFX6-NEXT:    v_mov_b32_e32 v2, s0
4969; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4970; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4971; GFX6-NEXT:    ds_write_b32 v0, v1
4972; GFX6-NEXT:    s_endpgm
4973;
4974; GFX7-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
4975; GFX7:       ; %bb.0: ; %entry
4976; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4977; GFX7-NEXT:    s_mov_b32 m0, -1
4978; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4979; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4980; GFX7-NEXT:    v_mov_b32_e32 v1, s2
4981; GFX7-NEXT:    v_mov_b32_e32 v2, s1
4982; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4983; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4984; GFX7-NEXT:    ds_write_b32 v0, v1
4985; GFX7-NEXT:    s_endpgm
4986;
4987; GFX10-WGP-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
4988; GFX10-WGP:       ; %bb.0: ; %entry
4989; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4990; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4991; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4992; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4993; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
4994; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
4995; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4996; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
4997; GFX10-WGP-NEXT:    s_endpgm
4998;
4999; GFX10-CU-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5000; GFX10-CU:       ; %bb.0: ; %entry
5001; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5002; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5003; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5004; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5005; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5006; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5007; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5008; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5009; GFX10-CU-NEXT:    s_endpgm
5010;
5011; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5012; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5013; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5014; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5015; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5016; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5017; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5018; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5019; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5020; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5021; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
5022; SKIP-CACHE-INV-NEXT:    s_endpgm
5023;
5024; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5025; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5026; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5027; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5028; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5029; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5030; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5031; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5032; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5033; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5034; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5035;
5036; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5037; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5038; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5039; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5040; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5041; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5042; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5043; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5044; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5045; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5046; GFX90A-TGSPLIT-NEXT:    s_endpgm
5047;
5048; GFX940-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5049; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5050; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5051; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5052; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5053; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5054; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5055; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5056; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5057; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5058; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5059;
5060; GFX940-TGSPLIT-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5061; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5062; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5063; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5064; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5065; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5066; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5067; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5068; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5069; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5070; GFX940-TGSPLIT-NEXT:    s_endpgm
5071;
5072; GFX11-WGP-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5073; GFX11-WGP:       ; %bb.0: ; %entry
5074; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5075; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5076; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5077; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
5078; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5079; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5080; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
5081; GFX11-WGP-NEXT:    s_endpgm
5082;
5083; GFX11-CU-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
5084; GFX11-CU:       ; %bb.0: ; %entry
5085; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5086; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5087; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5088; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
5089; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5090; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5091; GFX11-CU-NEXT:    ds_store_b32 v0, v1
5092; GFX11-CU-NEXT:    s_endpgm
5093    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5094entry:
5095  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5096  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
5097  %val0 = extractvalue { i32, i1 } %val, 0
5098  store i32 %val0, i32 addrspace(3)* %out, align 4
5099  ret void
5100}
5101
5102define amdgpu_kernel void @local_singlethread_monotonic_seq_cst_ret_cmpxchg(
5103; GFX6-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5104; GFX6:       ; %bb.0: ; %entry
5105; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5106; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5107; GFX6-NEXT:    s_mov_b32 m0, -1
5108; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5109; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5110; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5111; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5112; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5113; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5114; GFX6-NEXT:    ds_write_b32 v0, v1
5115; GFX6-NEXT:    s_endpgm
5116;
5117; GFX7-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5118; GFX7:       ; %bb.0: ; %entry
5119; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5120; GFX7-NEXT:    s_mov_b32 m0, -1
5121; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5122; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5123; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5124; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5125; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5126; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5127; GFX7-NEXT:    ds_write_b32 v0, v1
5128; GFX7-NEXT:    s_endpgm
5129;
5130; GFX10-WGP-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5131; GFX10-WGP:       ; %bb.0: ; %entry
5132; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5133; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5134; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5135; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5136; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5137; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5138; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5139; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
5140; GFX10-WGP-NEXT:    s_endpgm
5141;
5142; GFX10-CU-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5143; GFX10-CU:       ; %bb.0: ; %entry
5144; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5145; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5146; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5147; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5148; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5149; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5150; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5151; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5152; GFX10-CU-NEXT:    s_endpgm
5153;
5154; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5155; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5156; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5157; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5158; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5159; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5160; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5161; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5162; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5163; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5164; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
5165; SKIP-CACHE-INV-NEXT:    s_endpgm
5166;
5167; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5168; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5169; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5170; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5171; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5172; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5173; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5174; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5175; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5176; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5177; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5178;
5179; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5180; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5181; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5182; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5183; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5184; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5185; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5186; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5187; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5188; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5189; GFX90A-TGSPLIT-NEXT:    s_endpgm
5190;
5191; GFX940-NOTTGSPLIT-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5192; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5193; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5194; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5195; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5196; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5197; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5198; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5199; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5200; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5201; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5202;
5203; GFX940-TGSPLIT-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5204; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5205; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5206; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5207; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5208; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5209; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5210; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5211; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5212; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5213; GFX940-TGSPLIT-NEXT:    s_endpgm
5214;
5215; GFX11-WGP-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5216; GFX11-WGP:       ; %bb.0: ; %entry
5217; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5218; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5219; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5220; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
5221; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5222; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5223; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
5224; GFX11-WGP-NEXT:    s_endpgm
5225;
5226; GFX11-CU-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
5227; GFX11-CU:       ; %bb.0: ; %entry
5228; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5229; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5230; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5231; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
5232; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5233; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5234; GFX11-CU-NEXT:    ds_store_b32 v0, v1
5235; GFX11-CU-NEXT:    s_endpgm
5236    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5237entry:
5238  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5239  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic seq_cst
5240  %val0 = extractvalue { i32, i1 } %val, 0
5241  store i32 %val0, i32 addrspace(3)* %out, align 4
5242  ret void
5243}
5244
5245define amdgpu_kernel void @local_singlethread_acquire_seq_cst_ret_cmpxchg(
5246; GFX6-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5247; GFX6:       ; %bb.0: ; %entry
5248; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5249; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5250; GFX6-NEXT:    s_mov_b32 m0, -1
5251; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5252; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5253; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5254; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5255; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5256; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5257; GFX6-NEXT:    ds_write_b32 v0, v1
5258; GFX6-NEXT:    s_endpgm
5259;
5260; GFX7-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5261; GFX7:       ; %bb.0: ; %entry
5262; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5263; GFX7-NEXT:    s_mov_b32 m0, -1
5264; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5265; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5266; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5267; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5268; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5269; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5270; GFX7-NEXT:    ds_write_b32 v0, v1
5271; GFX7-NEXT:    s_endpgm
5272;
5273; GFX10-WGP-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5274; GFX10-WGP:       ; %bb.0: ; %entry
5275; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5276; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5277; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5278; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5279; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5280; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5281; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5282; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
5283; GFX10-WGP-NEXT:    s_endpgm
5284;
5285; GFX10-CU-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5286; GFX10-CU:       ; %bb.0: ; %entry
5287; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5288; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5289; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5290; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5291; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5292; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5293; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5294; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5295; GFX10-CU-NEXT:    s_endpgm
5296;
5297; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5298; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5299; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5300; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5301; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5302; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5303; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5304; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5305; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5306; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5307; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
5308; SKIP-CACHE-INV-NEXT:    s_endpgm
5309;
5310; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5311; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5312; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5313; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5314; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5315; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5316; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5317; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5318; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5319; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5320; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5321;
5322; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5323; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5324; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5325; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5326; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5327; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5328; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5329; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5330; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5331; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5332; GFX90A-TGSPLIT-NEXT:    s_endpgm
5333;
5334; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5335; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5336; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5337; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5338; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5339; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5340; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5341; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5342; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5343; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5344; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5345;
5346; GFX940-TGSPLIT-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5347; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5348; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5349; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5350; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5351; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5352; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5353; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5354; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5355; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5356; GFX940-TGSPLIT-NEXT:    s_endpgm
5357;
5358; GFX11-WGP-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5359; GFX11-WGP:       ; %bb.0: ; %entry
5360; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5361; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5362; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5363; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
5364; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5365; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5366; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
5367; GFX11-WGP-NEXT:    s_endpgm
5368;
5369; GFX11-CU-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
5370; GFX11-CU:       ; %bb.0: ; %entry
5371; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5372; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5373; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5374; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
5375; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5376; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5377; GFX11-CU-NEXT:    ds_store_b32 v0, v1
5378; GFX11-CU-NEXT:    s_endpgm
5379    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5380entry:
5381  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5382  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire seq_cst
5383  %val0 = extractvalue { i32, i1 } %val, 0
5384  store i32 %val0, i32 addrspace(3)* %out, align 4
5385  ret void
5386}
5387
5388define amdgpu_kernel void @local_singlethread_release_seq_cst_ret_cmpxchg(
5389; GFX6-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5390; GFX6:       ; %bb.0: ; %entry
5391; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5392; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5393; GFX6-NEXT:    s_mov_b32 m0, -1
5394; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5395; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5396; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5397; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5398; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5399; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5400; GFX6-NEXT:    ds_write_b32 v0, v1
5401; GFX6-NEXT:    s_endpgm
5402;
5403; GFX7-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5404; GFX7:       ; %bb.0: ; %entry
5405; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5406; GFX7-NEXT:    s_mov_b32 m0, -1
5407; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5408; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5409; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5410; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5411; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5412; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5413; GFX7-NEXT:    ds_write_b32 v0, v1
5414; GFX7-NEXT:    s_endpgm
5415;
5416; GFX10-WGP-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5417; GFX10-WGP:       ; %bb.0: ; %entry
5418; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5419; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5420; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5421; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5422; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5423; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5424; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5425; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
5426; GFX10-WGP-NEXT:    s_endpgm
5427;
5428; GFX10-CU-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5429; GFX10-CU:       ; %bb.0: ; %entry
5430; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5431; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5432; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5433; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5434; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5435; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5436; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5437; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5438; GFX10-CU-NEXT:    s_endpgm
5439;
5440; SKIP-CACHE-INV-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5441; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5442; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5443; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5444; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5445; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5446; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5447; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5448; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5449; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5450; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
5451; SKIP-CACHE-INV-NEXT:    s_endpgm
5452;
5453; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5454; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5455; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5456; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5457; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5458; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5459; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5460; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5461; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5462; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5463; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5464;
5465; GFX90A-TGSPLIT-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5466; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5467; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5468; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5469; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5470; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5471; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5472; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5473; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5474; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5475; GFX90A-TGSPLIT-NEXT:    s_endpgm
5476;
5477; GFX940-NOTTGSPLIT-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5478; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5479; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5480; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5481; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5482; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5483; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5484; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5485; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5486; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5487; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5488;
5489; GFX940-TGSPLIT-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5490; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5491; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5492; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5493; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5494; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5495; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5496; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5497; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5498; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5499; GFX940-TGSPLIT-NEXT:    s_endpgm
5500;
5501; GFX11-WGP-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5502; GFX11-WGP:       ; %bb.0: ; %entry
5503; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5504; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5505; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5506; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
5507; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5508; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5509; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
5510; GFX11-WGP-NEXT:    s_endpgm
5511;
5512; GFX11-CU-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
5513; GFX11-CU:       ; %bb.0: ; %entry
5514; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5515; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5516; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5517; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
5518; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5519; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5520; GFX11-CU-NEXT:    ds_store_b32 v0, v1
5521; GFX11-CU-NEXT:    s_endpgm
5522    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5523entry:
5524  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5525  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release seq_cst
5526  %val0 = extractvalue { i32, i1 } %val, 0
5527  store i32 %val0, i32 addrspace(3)* %out, align 4
5528  ret void
5529}
5530
5531define amdgpu_kernel void @local_singlethread_acq_rel_seq_cst_ret_cmpxchg(
5532; GFX6-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5533; GFX6:       ; %bb.0: ; %entry
5534; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5535; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5536; GFX6-NEXT:    s_mov_b32 m0, -1
5537; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5538; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5539; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5540; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5541; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5542; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5543; GFX6-NEXT:    ds_write_b32 v0, v1
5544; GFX6-NEXT:    s_endpgm
5545;
5546; GFX7-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5547; GFX7:       ; %bb.0: ; %entry
5548; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5549; GFX7-NEXT:    s_mov_b32 m0, -1
5550; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5551; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5552; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5553; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5554; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5555; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5556; GFX7-NEXT:    ds_write_b32 v0, v1
5557; GFX7-NEXT:    s_endpgm
5558;
5559; GFX10-WGP-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5560; GFX10-WGP:       ; %bb.0: ; %entry
5561; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5562; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5563; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5564; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5565; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5566; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5567; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5568; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
5569; GFX10-WGP-NEXT:    s_endpgm
5570;
5571; GFX10-CU-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5572; GFX10-CU:       ; %bb.0: ; %entry
5573; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5574; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5575; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5576; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5577; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5578; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5579; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5580; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5581; GFX10-CU-NEXT:    s_endpgm
5582;
5583; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5584; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5585; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5586; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5587; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5588; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5589; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5590; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5591; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5592; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5593; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
5594; SKIP-CACHE-INV-NEXT:    s_endpgm
5595;
5596; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5597; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5598; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5599; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5600; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5601; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5602; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5603; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5604; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5605; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5606; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5607;
5608; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5609; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5610; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5611; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5612; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5613; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5614; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5615; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5616; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5617; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5618; GFX90A-TGSPLIT-NEXT:    s_endpgm
5619;
5620; GFX940-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5621; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5622; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5623; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5624; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5625; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5626; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5627; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5628; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5629; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5630; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5631;
5632; GFX940-TGSPLIT-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5633; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5634; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5635; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5636; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5637; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5638; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5639; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5640; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5641; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5642; GFX940-TGSPLIT-NEXT:    s_endpgm
5643;
5644; GFX11-WGP-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5645; GFX11-WGP:       ; %bb.0: ; %entry
5646; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5647; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5648; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5649; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
5650; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5651; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5652; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
5653; GFX11-WGP-NEXT:    s_endpgm
5654;
5655; GFX11-CU-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5656; GFX11-CU:       ; %bb.0: ; %entry
5657; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5658; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5659; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5660; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
5661; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5662; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5663; GFX11-CU-NEXT:    ds_store_b32 v0, v1
5664; GFX11-CU-NEXT:    s_endpgm
5665    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5666entry:
5667  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5668  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel seq_cst
5669  %val0 = extractvalue { i32, i1 } %val, 0
5670  store i32 %val0, i32 addrspace(3)* %out, align 4
5671  ret void
5672}
5673
5674define amdgpu_kernel void @local_singlethread_seq_cst_seq_cst_ret_cmpxchg(
5675; GFX6-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5676; GFX6:       ; %bb.0: ; %entry
5677; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5678; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5679; GFX6-NEXT:    s_mov_b32 m0, -1
5680; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5681; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5682; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5683; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5684; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5685; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5686; GFX6-NEXT:    ds_write_b32 v0, v1
5687; GFX6-NEXT:    s_endpgm
5688;
5689; GFX7-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5690; GFX7:       ; %bb.0: ; %entry
5691; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5692; GFX7-NEXT:    s_mov_b32 m0, -1
5693; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5694; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5695; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5696; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5697; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5698; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5699; GFX7-NEXT:    ds_write_b32 v0, v1
5700; GFX7-NEXT:    s_endpgm
5701;
5702; GFX10-WGP-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5703; GFX10-WGP:       ; %bb.0: ; %entry
5704; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5705; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5706; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5707; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5708; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5709; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5710; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5711; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
5712; GFX10-WGP-NEXT:    s_endpgm
5713;
5714; GFX10-CU-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5715; GFX10-CU:       ; %bb.0: ; %entry
5716; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5717; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5718; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5719; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5720; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5721; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5722; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5723; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5724; GFX10-CU-NEXT:    s_endpgm
5725;
5726; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5727; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5728; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5729; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5730; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5731; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5732; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5733; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5734; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5735; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5736; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
5737; SKIP-CACHE-INV-NEXT:    s_endpgm
5738;
5739; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5740; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5741; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5742; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5743; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5744; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5745; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5746; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5747; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5748; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5749; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5750;
5751; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5752; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5753; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5754; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5755; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5756; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5757; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5758; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5759; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5760; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5761; GFX90A-TGSPLIT-NEXT:    s_endpgm
5762;
5763; GFX940-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5764; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5765; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5766; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5767; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5768; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5769; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5770; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5771; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5772; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5773; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5774;
5775; GFX940-TGSPLIT-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5776; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5777; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5778; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5779; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5780; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5781; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5782; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
5783; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5784; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5785; GFX940-TGSPLIT-NEXT:    s_endpgm
5786;
5787; GFX11-WGP-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5788; GFX11-WGP:       ; %bb.0: ; %entry
5789; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5790; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5791; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5792; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
5793; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5794; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5795; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
5796; GFX11-WGP-NEXT:    s_endpgm
5797;
5798; GFX11-CU-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5799; GFX11-CU:       ; %bb.0: ; %entry
5800; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
5801; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5802; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5803; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
5804; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
5805; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5806; GFX11-CU-NEXT:    ds_store_b32 v0, v1
5807; GFX11-CU-NEXT:    s_endpgm
5808    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5809entry:
5810  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5811  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
5812  %val0 = extractvalue { i32, i1 } %val, 0
5813  store i32 %val0, i32 addrspace(3)* %out, align 4
5814  ret void
5815}
5816
5817define amdgpu_kernel void @local_singlethread_one_as_unordered_load(
5818; GFX6-LABEL: local_singlethread_one_as_unordered_load:
5819; GFX6:       ; %bb.0: ; %entry
5820; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
5821; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
5822; GFX6-NEXT:    s_mov_b32 m0, -1
5823; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5824; GFX6-NEXT:    v_mov_b32_e32 v0, s0
5825; GFX6-NEXT:    ds_read_b32 v0, v0
5826; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5827; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5828; GFX6-NEXT:    ds_write_b32 v1, v0
5829; GFX6-NEXT:    s_endpgm
5830;
5831; GFX7-LABEL: local_singlethread_one_as_unordered_load:
5832; GFX7:       ; %bb.0: ; %entry
5833; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5834; GFX7-NEXT:    s_mov_b32 m0, -1
5835; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5836; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5837; GFX7-NEXT:    ds_read_b32 v0, v0
5838; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5839; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5840; GFX7-NEXT:    ds_write_b32 v1, v0
5841; GFX7-NEXT:    s_endpgm
5842;
5843; GFX10-WGP-LABEL: local_singlethread_one_as_unordered_load:
5844; GFX10-WGP:       ; %bb.0: ; %entry
5845; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5846; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5847; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5848; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5849; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
5850; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5851; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
5852; GFX10-WGP-NEXT:    s_endpgm
5853;
5854; GFX10-CU-LABEL: local_singlethread_one_as_unordered_load:
5855; GFX10-CU:       ; %bb.0: ; %entry
5856; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5857; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5858; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5859; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5860; GFX10-CU-NEXT:    ds_read_b32 v0, v0
5861; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5862; GFX10-CU-NEXT:    ds_write_b32 v1, v0
5863; GFX10-CU-NEXT:    s_endpgm
5864;
5865; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_unordered_load:
5866; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5867; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5868; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5869; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5870; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5871; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
5872; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
5873; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5874; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
5875; SKIP-CACHE-INV-NEXT:    s_endpgm
5876;
5877; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_unordered_load:
5878; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5879; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5880; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5881; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5882; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
5883; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5884; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5885; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
5886; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5887;
5888; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_unordered_load:
5889; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5890; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5891; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5892; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5893; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
5894; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5895; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5896; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
5897; GFX90A-TGSPLIT-NEXT:    s_endpgm
5898;
5899; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_unordered_load:
5900; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5901; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5902; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5903; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5904; GFX940-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
5905; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5906; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5907; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
5908; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5909;
5910; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_unordered_load:
5911; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5912; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5913; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5914; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5915; GFX940-TGSPLIT-NEXT:    ds_read_b32 v0, v0
5916; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5917; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5918; GFX940-TGSPLIT-NEXT:    ds_write_b32 v1, v0
5919; GFX940-TGSPLIT-NEXT:    s_endpgm
5920;
5921; GFX11-WGP-LABEL: local_singlethread_one_as_unordered_load:
5922; GFX11-WGP:       ; %bb.0: ; %entry
5923; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5924; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5925; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5926; GFX11-WGP-NEXT:    ds_load_b32 v0, v0
5927; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5928; GFX11-WGP-NEXT:    ds_store_b32 v1, v0
5929; GFX11-WGP-NEXT:    s_endpgm
5930;
5931; GFX11-CU-LABEL: local_singlethread_one_as_unordered_load:
5932; GFX11-CU:       ; %bb.0: ; %entry
5933; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5934; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5935; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5936; GFX11-CU-NEXT:    ds_load_b32 v0, v0
5937; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5938; GFX11-CU-NEXT:    ds_store_b32 v1, v0
5939; GFX11-CU-NEXT:    s_endpgm
5940    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
5941entry:
5942  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread-one-as") unordered, align 4
5943  store i32 %val, i32 addrspace(3)* %out
5944  ret void
5945}
5946
5947define amdgpu_kernel void @local_singlethread_one_as_monotonic_load(
5948; GFX6-LABEL: local_singlethread_one_as_monotonic_load:
5949; GFX6:       ; %bb.0: ; %entry
5950; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
5951; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
5952; GFX6-NEXT:    s_mov_b32 m0, -1
5953; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5954; GFX6-NEXT:    v_mov_b32_e32 v0, s0
5955; GFX6-NEXT:    ds_read_b32 v0, v0
5956; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5957; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5958; GFX6-NEXT:    ds_write_b32 v1, v0
5959; GFX6-NEXT:    s_endpgm
5960;
5961; GFX7-LABEL: local_singlethread_one_as_monotonic_load:
5962; GFX7:       ; %bb.0: ; %entry
5963; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5964; GFX7-NEXT:    s_mov_b32 m0, -1
5965; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5966; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5967; GFX7-NEXT:    ds_read_b32 v0, v0
5968; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5969; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5970; GFX7-NEXT:    ds_write_b32 v1, v0
5971; GFX7-NEXT:    s_endpgm
5972;
5973; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_load:
5974; GFX10-WGP:       ; %bb.0: ; %entry
5975; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5976; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5977; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5978; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5979; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
5980; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5981; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
5982; GFX10-WGP-NEXT:    s_endpgm
5983;
5984; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_load:
5985; GFX10-CU:       ; %bb.0: ; %entry
5986; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5987; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5988; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5989; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5990; GFX10-CU-NEXT:    ds_read_b32 v0, v0
5991; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5992; GFX10-CU-NEXT:    ds_write_b32 v1, v0
5993; GFX10-CU-NEXT:    s_endpgm
5994;
5995; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_load:
5996; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5997; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5998; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5999; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6000; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6001; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
6002; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6003; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6004; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
6005; SKIP-CACHE-INV-NEXT:    s_endpgm
6006;
6007; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_load:
6008; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6009; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6010; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6011; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6012; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
6013; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6014; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6015; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
6016; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6017;
6018; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_load:
6019; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6020; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6021; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6022; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6023; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
6024; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6025; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6026; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
6027; GFX90A-TGSPLIT-NEXT:    s_endpgm
6028;
6029; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_load:
6030; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6031; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6032; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6033; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6034; GFX940-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
6035; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6036; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6037; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
6038; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6039;
6040; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_load:
6041; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6042; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6043; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6044; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6045; GFX940-TGSPLIT-NEXT:    ds_read_b32 v0, v0
6046; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6047; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6048; GFX940-TGSPLIT-NEXT:    ds_write_b32 v1, v0
6049; GFX940-TGSPLIT-NEXT:    s_endpgm
6050;
6051; GFX11-WGP-LABEL: local_singlethread_one_as_monotonic_load:
6052; GFX11-WGP:       ; %bb.0: ; %entry
6053; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6054; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6055; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6056; GFX11-WGP-NEXT:    ds_load_b32 v0, v0
6057; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6058; GFX11-WGP-NEXT:    ds_store_b32 v1, v0
6059; GFX11-WGP-NEXT:    s_endpgm
6060;
6061; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_load:
6062; GFX11-CU:       ; %bb.0: ; %entry
6063; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6064; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6065; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6066; GFX11-CU-NEXT:    ds_load_b32 v0, v0
6067; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6068; GFX11-CU-NEXT:    ds_store_b32 v1, v0
6069; GFX11-CU-NEXT:    s_endpgm
6070    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
6071entry:
6072  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread-one-as") monotonic, align 4
6073  store i32 %val, i32 addrspace(3)* %out
6074  ret void
6075}
6076
6077define amdgpu_kernel void @local_singlethread_one_as_acquire_load(
6078; GFX6-LABEL: local_singlethread_one_as_acquire_load:
6079; GFX6:       ; %bb.0: ; %entry
6080; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
6081; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
6082; GFX6-NEXT:    s_mov_b32 m0, -1
6083; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6084; GFX6-NEXT:    v_mov_b32_e32 v0, s0
6085; GFX6-NEXT:    ds_read_b32 v0, v0
6086; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6087; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6088; GFX6-NEXT:    ds_write_b32 v1, v0
6089; GFX6-NEXT:    s_endpgm
6090;
6091; GFX7-LABEL: local_singlethread_one_as_acquire_load:
6092; GFX7:       ; %bb.0: ; %entry
6093; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6094; GFX7-NEXT:    s_mov_b32 m0, -1
6095; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6096; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6097; GFX7-NEXT:    ds_read_b32 v0, v0
6098; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6099; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6100; GFX7-NEXT:    ds_write_b32 v1, v0
6101; GFX7-NEXT:    s_endpgm
6102;
6103; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_load:
6104; GFX10-WGP:       ; %bb.0: ; %entry
6105; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6106; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6107; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6108; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6109; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
6110; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6111; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
6112; GFX10-WGP-NEXT:    s_endpgm
6113;
6114; GFX10-CU-LABEL: local_singlethread_one_as_acquire_load:
6115; GFX10-CU:       ; %bb.0: ; %entry
6116; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6117; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6118; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6119; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6120; GFX10-CU-NEXT:    ds_read_b32 v0, v0
6121; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6122; GFX10-CU-NEXT:    ds_write_b32 v1, v0
6123; GFX10-CU-NEXT:    s_endpgm
6124;
6125; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_load:
6126; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6127; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6128; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6129; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6130; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6131; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
6132; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6133; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6134; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
6135; SKIP-CACHE-INV-NEXT:    s_endpgm
6136;
6137; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_load:
6138; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6139; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6140; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6141; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6142; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
6143; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6144; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6145; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
6146; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6147;
6148; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_load:
6149; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6150; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6151; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6152; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6153; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
6154; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6155; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6156; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
6157; GFX90A-TGSPLIT-NEXT:    s_endpgm
6158;
6159; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_load:
6160; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6161; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6162; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6163; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6164; GFX940-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
6165; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6166; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6167; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
6168; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6169;
6170; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acquire_load:
6171; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6172; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6173; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6174; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6175; GFX940-TGSPLIT-NEXT:    ds_read_b32 v0, v0
6176; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6177; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6178; GFX940-TGSPLIT-NEXT:    ds_write_b32 v1, v0
6179; GFX940-TGSPLIT-NEXT:    s_endpgm
6180;
6181; GFX11-WGP-LABEL: local_singlethread_one_as_acquire_load:
6182; GFX11-WGP:       ; %bb.0: ; %entry
6183; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6184; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6185; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6186; GFX11-WGP-NEXT:    ds_load_b32 v0, v0
6187; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6188; GFX11-WGP-NEXT:    ds_store_b32 v1, v0
6189; GFX11-WGP-NEXT:    s_endpgm
6190;
6191; GFX11-CU-LABEL: local_singlethread_one_as_acquire_load:
6192; GFX11-CU:       ; %bb.0: ; %entry
6193; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6194; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6195; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6196; GFX11-CU-NEXT:    ds_load_b32 v0, v0
6197; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6198; GFX11-CU-NEXT:    ds_store_b32 v1, v0
6199; GFX11-CU-NEXT:    s_endpgm
6200    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
6201entry:
6202  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread-one-as") acquire, align 4
6203  store i32 %val, i32 addrspace(3)* %out
6204  ret void
6205}
6206
6207define amdgpu_kernel void @local_singlethread_one_as_seq_cst_load(
6208; GFX6-LABEL: local_singlethread_one_as_seq_cst_load:
6209; GFX6:       ; %bb.0: ; %entry
6210; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
6211; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
6212; GFX6-NEXT:    s_mov_b32 m0, -1
6213; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6214; GFX6-NEXT:    v_mov_b32_e32 v0, s0
6215; GFX6-NEXT:    ds_read_b32 v0, v0
6216; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6217; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6218; GFX6-NEXT:    ds_write_b32 v1, v0
6219; GFX6-NEXT:    s_endpgm
6220;
6221; GFX7-LABEL: local_singlethread_one_as_seq_cst_load:
6222; GFX7:       ; %bb.0: ; %entry
6223; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6224; GFX7-NEXT:    s_mov_b32 m0, -1
6225; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6226; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6227; GFX7-NEXT:    ds_read_b32 v0, v0
6228; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6229; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6230; GFX7-NEXT:    ds_write_b32 v1, v0
6231; GFX7-NEXT:    s_endpgm
6232;
6233; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_load:
6234; GFX10-WGP:       ; %bb.0: ; %entry
6235; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6236; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6237; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6238; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6239; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
6240; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6241; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
6242; GFX10-WGP-NEXT:    s_endpgm
6243;
6244; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_load:
6245; GFX10-CU:       ; %bb.0: ; %entry
6246; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6247; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6248; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6249; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6250; GFX10-CU-NEXT:    ds_read_b32 v0, v0
6251; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6252; GFX10-CU-NEXT:    ds_write_b32 v1, v0
6253; GFX10-CU-NEXT:    s_endpgm
6254;
6255; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_load:
6256; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6257; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6258; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6259; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6260; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6261; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
6262; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6263; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6264; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
6265; SKIP-CACHE-INV-NEXT:    s_endpgm
6266;
6267; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_load:
6268; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6269; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6270; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6271; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6272; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
6273; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6274; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6275; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
6276; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6277;
6278; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_load:
6279; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6280; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6281; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6282; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6283; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
6284; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6285; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6286; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
6287; GFX90A-TGSPLIT-NEXT:    s_endpgm
6288;
6289; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_load:
6290; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6291; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6292; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6293; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6294; GFX940-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
6295; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6296; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6297; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
6298; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6299;
6300; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_load:
6301; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6302; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6303; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6304; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6305; GFX940-TGSPLIT-NEXT:    ds_read_b32 v0, v0
6306; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6307; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6308; GFX940-TGSPLIT-NEXT:    ds_write_b32 v1, v0
6309; GFX940-TGSPLIT-NEXT:    s_endpgm
6310;
6311; GFX11-WGP-LABEL: local_singlethread_one_as_seq_cst_load:
6312; GFX11-WGP:       ; %bb.0: ; %entry
6313; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6314; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6315; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6316; GFX11-WGP-NEXT:    ds_load_b32 v0, v0
6317; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6318; GFX11-WGP-NEXT:    ds_store_b32 v1, v0
6319; GFX11-WGP-NEXT:    s_endpgm
6320;
6321; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_load:
6322; GFX11-CU:       ; %bb.0: ; %entry
6323; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6324; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6325; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6326; GFX11-CU-NEXT:    ds_load_b32 v0, v0
6327; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6328; GFX11-CU-NEXT:    ds_store_b32 v1, v0
6329; GFX11-CU-NEXT:    s_endpgm
6330    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
6331entry:
6332  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread-one-as") seq_cst, align 4
6333  store i32 %val, i32 addrspace(3)* %out
6334  ret void
6335}
6336
6337define amdgpu_kernel void @local_singlethread_one_as_unordered_store(
6338; GFX6-LABEL: local_singlethread_one_as_unordered_store:
6339; GFX6:       ; %bb.0: ; %entry
6340; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
6341; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
6342; GFX6-NEXT:    s_mov_b32 m0, -1
6343; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6344; GFX6-NEXT:    v_mov_b32_e32 v0, s0
6345; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6346; GFX6-NEXT:    ds_write_b32 v0, v1
6347; GFX6-NEXT:    s_endpgm
6348;
6349; GFX7-LABEL: local_singlethread_one_as_unordered_store:
6350; GFX7:       ; %bb.0: ; %entry
6351; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6352; GFX7-NEXT:    s_mov_b32 m0, -1
6353; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6354; GFX7-NEXT:    v_mov_b32_e32 v0, s1
6355; GFX7-NEXT:    v_mov_b32_e32 v1, s0
6356; GFX7-NEXT:    ds_write_b32 v0, v1
6357; GFX7-NEXT:    s_endpgm
6358;
6359; GFX10-WGP-LABEL: local_singlethread_one_as_unordered_store:
6360; GFX10-WGP:       ; %bb.0: ; %entry
6361; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6362; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6363; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
6364; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
6365; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
6366; GFX10-WGP-NEXT:    s_endpgm
6367;
6368; GFX10-CU-LABEL: local_singlethread_one_as_unordered_store:
6369; GFX10-CU:       ; %bb.0: ; %entry
6370; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6371; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6372; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
6373; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
6374; GFX10-CU-NEXT:    ds_write_b32 v0, v1
6375; GFX10-CU-NEXT:    s_endpgm
6376;
6377; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_unordered_store:
6378; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6379; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6380; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6381; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6382; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
6383; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
6384; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
6385; SKIP-CACHE-INV-NEXT:    s_endpgm
6386;
6387; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_unordered_store:
6388; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6389; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6390; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6391; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6392; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6393; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6394; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6395;
6396; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_unordered_store:
6397; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6398; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6399; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6400; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6401; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6402; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6403; GFX90A-TGSPLIT-NEXT:    s_endpgm
6404;
6405; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_unordered_store:
6406; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6407; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6408; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6409; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6410; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6411; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6412; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6413;
6414; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_unordered_store:
6415; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6416; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6417; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6418; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6419; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6420; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6421; GFX940-TGSPLIT-NEXT:    s_endpgm
6422;
6423; GFX11-WGP-LABEL: local_singlethread_one_as_unordered_store:
6424; GFX11-WGP:       ; %bb.0: ; %entry
6425; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6426; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6427; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
6428; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
6429; GFX11-WGP-NEXT:    s_endpgm
6430;
6431; GFX11-CU-LABEL: local_singlethread_one_as_unordered_store:
6432; GFX11-CU:       ; %bb.0: ; %entry
6433; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6434; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6435; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
6436; GFX11-CU-NEXT:    ds_store_b32 v0, v1
6437; GFX11-CU-NEXT:    s_endpgm
6438    i32 %in, i32 addrspace(3)* %out) {
6439entry:
6440  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread-one-as") unordered, align 4
6441  ret void
6442}
6443
6444define amdgpu_kernel void @local_singlethread_one_as_monotonic_store(
6445; GFX6-LABEL: local_singlethread_one_as_monotonic_store:
6446; GFX6:       ; %bb.0: ; %entry
6447; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
6448; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
6449; GFX6-NEXT:    s_mov_b32 m0, -1
6450; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6451; GFX6-NEXT:    v_mov_b32_e32 v0, s0
6452; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6453; GFX6-NEXT:    ds_write_b32 v0, v1
6454; GFX6-NEXT:    s_endpgm
6455;
6456; GFX7-LABEL: local_singlethread_one_as_monotonic_store:
6457; GFX7:       ; %bb.0: ; %entry
6458; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6459; GFX7-NEXT:    s_mov_b32 m0, -1
6460; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6461; GFX7-NEXT:    v_mov_b32_e32 v0, s1
6462; GFX7-NEXT:    v_mov_b32_e32 v1, s0
6463; GFX7-NEXT:    ds_write_b32 v0, v1
6464; GFX7-NEXT:    s_endpgm
6465;
6466; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_store:
6467; GFX10-WGP:       ; %bb.0: ; %entry
6468; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6469; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6470; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
6471; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
6472; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
6473; GFX10-WGP-NEXT:    s_endpgm
6474;
6475; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_store:
6476; GFX10-CU:       ; %bb.0: ; %entry
6477; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6478; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6479; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
6480; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
6481; GFX10-CU-NEXT:    ds_write_b32 v0, v1
6482; GFX10-CU-NEXT:    s_endpgm
6483;
6484; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_store:
6485; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6486; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6487; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6488; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6489; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
6490; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
6491; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
6492; SKIP-CACHE-INV-NEXT:    s_endpgm
6493;
6494; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_store:
6495; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6496; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6497; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6498; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6499; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6500; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6501; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6502;
6503; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_store:
6504; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6505; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6506; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6507; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6508; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6509; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6510; GFX90A-TGSPLIT-NEXT:    s_endpgm
6511;
6512; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_store:
6513; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6514; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6515; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6516; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6517; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6518; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6519; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6520;
6521; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_store:
6522; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6523; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6524; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6525; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6526; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6527; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6528; GFX940-TGSPLIT-NEXT:    s_endpgm
6529;
6530; GFX11-WGP-LABEL: local_singlethread_one_as_monotonic_store:
6531; GFX11-WGP:       ; %bb.0: ; %entry
6532; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6533; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6534; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
6535; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
6536; GFX11-WGP-NEXT:    s_endpgm
6537;
6538; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_store:
6539; GFX11-CU:       ; %bb.0: ; %entry
6540; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6541; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6542; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
6543; GFX11-CU-NEXT:    ds_store_b32 v0, v1
6544; GFX11-CU-NEXT:    s_endpgm
6545    i32 %in, i32 addrspace(3)* %out) {
6546entry:
6547  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread-one-as") monotonic, align 4
6548  ret void
6549}
6550
6551define amdgpu_kernel void @local_singlethread_one_as_release_store(
6552; GFX6-LABEL: local_singlethread_one_as_release_store:
6553; GFX6:       ; %bb.0: ; %entry
6554; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
6555; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
6556; GFX6-NEXT:    s_mov_b32 m0, -1
6557; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6558; GFX6-NEXT:    v_mov_b32_e32 v0, s0
6559; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6560; GFX6-NEXT:    ds_write_b32 v0, v1
6561; GFX6-NEXT:    s_endpgm
6562;
6563; GFX7-LABEL: local_singlethread_one_as_release_store:
6564; GFX7:       ; %bb.0: ; %entry
6565; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6566; GFX7-NEXT:    s_mov_b32 m0, -1
6567; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6568; GFX7-NEXT:    v_mov_b32_e32 v0, s1
6569; GFX7-NEXT:    v_mov_b32_e32 v1, s0
6570; GFX7-NEXT:    ds_write_b32 v0, v1
6571; GFX7-NEXT:    s_endpgm
6572;
6573; GFX10-WGP-LABEL: local_singlethread_one_as_release_store:
6574; GFX10-WGP:       ; %bb.0: ; %entry
6575; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6576; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6577; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
6578; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
6579; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
6580; GFX10-WGP-NEXT:    s_endpgm
6581;
6582; GFX10-CU-LABEL: local_singlethread_one_as_release_store:
6583; GFX10-CU:       ; %bb.0: ; %entry
6584; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6585; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6586; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
6587; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
6588; GFX10-CU-NEXT:    ds_write_b32 v0, v1
6589; GFX10-CU-NEXT:    s_endpgm
6590;
6591; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_store:
6592; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6593; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6594; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6595; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6596; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
6597; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
6598; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
6599; SKIP-CACHE-INV-NEXT:    s_endpgm
6600;
6601; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_store:
6602; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6603; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6604; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6605; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6606; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6607; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6608; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6609;
6610; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_store:
6611; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6612; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6613; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6614; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6615; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6616; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6617; GFX90A-TGSPLIT-NEXT:    s_endpgm
6618;
6619; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_store:
6620; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6621; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6622; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6623; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6624; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6625; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6626; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6627;
6628; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_release_store:
6629; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6630; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6631; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6632; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6633; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6634; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6635; GFX940-TGSPLIT-NEXT:    s_endpgm
6636;
6637; GFX11-WGP-LABEL: local_singlethread_one_as_release_store:
6638; GFX11-WGP:       ; %bb.0: ; %entry
6639; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6640; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6641; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
6642; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
6643; GFX11-WGP-NEXT:    s_endpgm
6644;
6645; GFX11-CU-LABEL: local_singlethread_one_as_release_store:
6646; GFX11-CU:       ; %bb.0: ; %entry
6647; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6648; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6649; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
6650; GFX11-CU-NEXT:    ds_store_b32 v0, v1
6651; GFX11-CU-NEXT:    s_endpgm
6652    i32 %in, i32 addrspace(3)* %out) {
6653entry:
6654  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread-one-as") release, align 4
6655  ret void
6656}
6657
6658define amdgpu_kernel void @local_singlethread_one_as_seq_cst_store(
6659; GFX6-LABEL: local_singlethread_one_as_seq_cst_store:
6660; GFX6:       ; %bb.0: ; %entry
6661; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
6662; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
6663; GFX6-NEXT:    s_mov_b32 m0, -1
6664; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6665; GFX6-NEXT:    v_mov_b32_e32 v0, s0
6666; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6667; GFX6-NEXT:    ds_write_b32 v0, v1
6668; GFX6-NEXT:    s_endpgm
6669;
6670; GFX7-LABEL: local_singlethread_one_as_seq_cst_store:
6671; GFX7:       ; %bb.0: ; %entry
6672; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6673; GFX7-NEXT:    s_mov_b32 m0, -1
6674; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6675; GFX7-NEXT:    v_mov_b32_e32 v0, s1
6676; GFX7-NEXT:    v_mov_b32_e32 v1, s0
6677; GFX7-NEXT:    ds_write_b32 v0, v1
6678; GFX7-NEXT:    s_endpgm
6679;
6680; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_store:
6681; GFX10-WGP:       ; %bb.0: ; %entry
6682; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6683; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6684; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
6685; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
6686; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
6687; GFX10-WGP-NEXT:    s_endpgm
6688;
6689; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_store:
6690; GFX10-CU:       ; %bb.0: ; %entry
6691; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6692; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6693; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
6694; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
6695; GFX10-CU-NEXT:    ds_write_b32 v0, v1
6696; GFX10-CU-NEXT:    s_endpgm
6697;
6698; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_store:
6699; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6700; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6701; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6702; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6703; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
6704; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
6705; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
6706; SKIP-CACHE-INV-NEXT:    s_endpgm
6707;
6708; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_store:
6709; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6710; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6711; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6712; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6713; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6714; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6715; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6716;
6717; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_store:
6718; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6719; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6720; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6721; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6722; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6723; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6724; GFX90A-TGSPLIT-NEXT:    s_endpgm
6725;
6726; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_store:
6727; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6728; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6729; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6730; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6731; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6732; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6733; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6734;
6735; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_store:
6736; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6737; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6738; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6739; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
6740; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
6741; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6742; GFX940-TGSPLIT-NEXT:    s_endpgm
6743;
6744; GFX11-WGP-LABEL: local_singlethread_one_as_seq_cst_store:
6745; GFX11-WGP:       ; %bb.0: ; %entry
6746; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6747; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6748; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
6749; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
6750; GFX11-WGP-NEXT:    s_endpgm
6751;
6752; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_store:
6753; GFX11-CU:       ; %bb.0: ; %entry
6754; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6755; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6756; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
6757; GFX11-CU-NEXT:    ds_store_b32 v0, v1
6758; GFX11-CU-NEXT:    s_endpgm
6759    i32 %in, i32 addrspace(3)* %out) {
6760entry:
6761  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread-one-as") seq_cst, align 4
6762  ret void
6763}
6764
6765define amdgpu_kernel void @local_singlethread_one_as_monotonic_atomicrmw(
6766; GFX6-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6767; GFX6:       ; %bb.0: ; %entry
6768; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
6769; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
6770; GFX6-NEXT:    s_mov_b32 m0, -1
6771; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6772; GFX6-NEXT:    v_mov_b32_e32 v0, s0
6773; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6774; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6775; GFX6-NEXT:    s_endpgm
6776;
6777; GFX7-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6778; GFX7:       ; %bb.0: ; %entry
6779; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6780; GFX7-NEXT:    s_mov_b32 m0, -1
6781; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6782; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6783; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6784; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6785; GFX7-NEXT:    s_endpgm
6786;
6787; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6788; GFX10-WGP:       ; %bb.0: ; %entry
6789; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6790; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6791; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6792; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6793; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6794; GFX10-WGP-NEXT:    s_endpgm
6795;
6796; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6797; GFX10-CU:       ; %bb.0: ; %entry
6798; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6799; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6800; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6801; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6802; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6803; GFX10-CU-NEXT:    s_endpgm
6804;
6805; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6806; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6807; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6808; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6809; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6810; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6811; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6812; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6813; SKIP-CACHE-INV-NEXT:    s_endpgm
6814;
6815; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6816; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6817; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6818; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6819; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6820; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6821; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6822; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6823;
6824; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6825; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6826; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6827; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6828; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6829; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6830; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6831; GFX90A-TGSPLIT-NEXT:    s_endpgm
6832;
6833; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6834; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6835; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6836; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6837; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6838; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6839; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6840; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6841;
6842; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6843; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6844; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6845; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6846; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6847; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6848; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6849; GFX940-TGSPLIT-NEXT:    s_endpgm
6850;
6851; GFX11-WGP-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6852; GFX11-WGP:       ; %bb.0: ; %entry
6853; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6854; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6855; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6856; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
6857; GFX11-WGP-NEXT:    s_endpgm
6858;
6859; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
6860; GFX11-CU:       ; %bb.0: ; %entry
6861; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6862; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6863; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6864; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
6865; GFX11-CU-NEXT:    s_endpgm
6866    i32 addrspace(3)* %out, i32 %in) {
6867entry:
6868  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") monotonic
6869  ret void
6870}
6871
6872define amdgpu_kernel void @local_singlethread_one_as_acquire_atomicrmw(
6873; GFX6-LABEL: local_singlethread_one_as_acquire_atomicrmw:
6874; GFX6:       ; %bb.0: ; %entry
6875; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
6876; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
6877; GFX6-NEXT:    s_mov_b32 m0, -1
6878; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6879; GFX6-NEXT:    v_mov_b32_e32 v0, s0
6880; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6881; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6882; GFX6-NEXT:    s_endpgm
6883;
6884; GFX7-LABEL: local_singlethread_one_as_acquire_atomicrmw:
6885; GFX7:       ; %bb.0: ; %entry
6886; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6887; GFX7-NEXT:    s_mov_b32 m0, -1
6888; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6889; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6890; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6891; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6892; GFX7-NEXT:    s_endpgm
6893;
6894; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_atomicrmw:
6895; GFX10-WGP:       ; %bb.0: ; %entry
6896; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6897; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6898; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6899; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6900; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6901; GFX10-WGP-NEXT:    s_endpgm
6902;
6903; GFX10-CU-LABEL: local_singlethread_one_as_acquire_atomicrmw:
6904; GFX10-CU:       ; %bb.0: ; %entry
6905; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6906; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6907; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6908; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6909; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6910; GFX10-CU-NEXT:    s_endpgm
6911;
6912; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_atomicrmw:
6913; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6914; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6915; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6916; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6917; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6918; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6919; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6920; SKIP-CACHE-INV-NEXT:    s_endpgm
6921;
6922; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_atomicrmw:
6923; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6924; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6925; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6926; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6927; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6928; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6929; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6930;
6931; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_atomicrmw:
6932; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6933; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6934; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6935; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6936; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6937; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6938; GFX90A-TGSPLIT-NEXT:    s_endpgm
6939;
6940; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_atomicrmw:
6941; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6942; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6943; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6944; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6945; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6946; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6947; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6948;
6949; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acquire_atomicrmw:
6950; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6951; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6952; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6953; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6954; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6955; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6956; GFX940-TGSPLIT-NEXT:    s_endpgm
6957;
6958; GFX11-WGP-LABEL: local_singlethread_one_as_acquire_atomicrmw:
6959; GFX11-WGP:       ; %bb.0: ; %entry
6960; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6961; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6962; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6963; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
6964; GFX11-WGP-NEXT:    s_endpgm
6965;
6966; GFX11-CU-LABEL: local_singlethread_one_as_acquire_atomicrmw:
6967; GFX11-CU:       ; %bb.0: ; %entry
6968; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6969; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6970; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6971; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
6972; GFX11-CU-NEXT:    s_endpgm
6973    i32 addrspace(3)* %out, i32 %in) {
6974entry:
6975  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") acquire
6976  ret void
6977}
6978
6979define amdgpu_kernel void @local_singlethread_one_as_release_atomicrmw(
6980; GFX6-LABEL: local_singlethread_one_as_release_atomicrmw:
6981; GFX6:       ; %bb.0: ; %entry
6982; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
6983; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
6984; GFX6-NEXT:    s_mov_b32 m0, -1
6985; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6986; GFX6-NEXT:    v_mov_b32_e32 v0, s0
6987; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6988; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6989; GFX6-NEXT:    s_endpgm
6990;
6991; GFX7-LABEL: local_singlethread_one_as_release_atomicrmw:
6992; GFX7:       ; %bb.0: ; %entry
6993; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6994; GFX7-NEXT:    s_mov_b32 m0, -1
6995; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6996; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6997; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6998; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
6999; GFX7-NEXT:    s_endpgm
7000;
7001; GFX10-WGP-LABEL: local_singlethread_one_as_release_atomicrmw:
7002; GFX10-WGP:       ; %bb.0: ; %entry
7003; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7004; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7005; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7006; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7007; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7008; GFX10-WGP-NEXT:    s_endpgm
7009;
7010; GFX10-CU-LABEL: local_singlethread_one_as_release_atomicrmw:
7011; GFX10-CU:       ; %bb.0: ; %entry
7012; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7013; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7014; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7015; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7016; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7017; GFX10-CU-NEXT:    s_endpgm
7018;
7019; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_atomicrmw:
7020; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7021; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7022; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7023; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7024; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7025; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7026; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7027; SKIP-CACHE-INV-NEXT:    s_endpgm
7028;
7029; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_atomicrmw:
7030; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7031; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7032; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7033; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7034; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7035; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7036; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7037;
7038; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_atomicrmw:
7039; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7040; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7041; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7042; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7043; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7044; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7045; GFX90A-TGSPLIT-NEXT:    s_endpgm
7046;
7047; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_atomicrmw:
7048; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7049; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7050; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7051; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7052; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7053; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7054; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7055;
7056; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_release_atomicrmw:
7057; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7058; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7059; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7060; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7061; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7062; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7063; GFX940-TGSPLIT-NEXT:    s_endpgm
7064;
7065; GFX11-WGP-LABEL: local_singlethread_one_as_release_atomicrmw:
7066; GFX11-WGP:       ; %bb.0: ; %entry
7067; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7068; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7069; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7070; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
7071; GFX11-WGP-NEXT:    s_endpgm
7072;
7073; GFX11-CU-LABEL: local_singlethread_one_as_release_atomicrmw:
7074; GFX11-CU:       ; %bb.0: ; %entry
7075; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7076; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7077; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7078; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
7079; GFX11-CU-NEXT:    s_endpgm
7080    i32 addrspace(3)* %out, i32 %in) {
7081entry:
7082  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") release
7083  ret void
7084}
7085
7086define amdgpu_kernel void @local_singlethread_one_as_acq_rel_atomicrmw(
7087; GFX6-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7088; GFX6:       ; %bb.0: ; %entry
7089; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
7090; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
7091; GFX6-NEXT:    s_mov_b32 m0, -1
7092; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7093; GFX6-NEXT:    v_mov_b32_e32 v0, s0
7094; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7095; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7096; GFX6-NEXT:    s_endpgm
7097;
7098; GFX7-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7099; GFX7:       ; %bb.0: ; %entry
7100; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7101; GFX7-NEXT:    s_mov_b32 m0, -1
7102; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7103; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7104; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7105; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7106; GFX7-NEXT:    s_endpgm
7107;
7108; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7109; GFX10-WGP:       ; %bb.0: ; %entry
7110; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7111; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7112; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7113; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7114; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7115; GFX10-WGP-NEXT:    s_endpgm
7116;
7117; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7118; GFX10-CU:       ; %bb.0: ; %entry
7119; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7120; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7121; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7122; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7123; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7124; GFX10-CU-NEXT:    s_endpgm
7125;
7126; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7127; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7128; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7129; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7130; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7131; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7132; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7133; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7134; SKIP-CACHE-INV-NEXT:    s_endpgm
7135;
7136; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7137; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7138; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7139; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7140; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7141; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7142; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7143; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7144;
7145; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7146; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7147; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7148; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7149; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7150; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7151; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7152; GFX90A-TGSPLIT-NEXT:    s_endpgm
7153;
7154; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7155; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7156; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7157; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7158; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7159; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7160; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7161; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7162;
7163; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7164; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7165; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7166; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7167; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7168; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7169; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7170; GFX940-TGSPLIT-NEXT:    s_endpgm
7171;
7172; GFX11-WGP-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7173; GFX11-WGP:       ; %bb.0: ; %entry
7174; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7175; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7176; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7177; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
7178; GFX11-WGP-NEXT:    s_endpgm
7179;
7180; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
7181; GFX11-CU:       ; %bb.0: ; %entry
7182; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7183; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7184; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7185; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
7186; GFX11-CU-NEXT:    s_endpgm
7187    i32 addrspace(3)* %out, i32 %in) {
7188entry:
7189  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") acq_rel
7190  ret void
7191}
7192
7193define amdgpu_kernel void @local_singlethread_one_as_seq_cst_atomicrmw(
7194; GFX6-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7195; GFX6:       ; %bb.0: ; %entry
7196; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
7197; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
7198; GFX6-NEXT:    s_mov_b32 m0, -1
7199; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7200; GFX6-NEXT:    v_mov_b32_e32 v0, s0
7201; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7202; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7203; GFX6-NEXT:    s_endpgm
7204;
7205; GFX7-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7206; GFX7:       ; %bb.0: ; %entry
7207; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7208; GFX7-NEXT:    s_mov_b32 m0, -1
7209; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7210; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7211; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7212; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7213; GFX7-NEXT:    s_endpgm
7214;
7215; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7216; GFX10-WGP:       ; %bb.0: ; %entry
7217; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7218; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7219; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7220; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7221; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7222; GFX10-WGP-NEXT:    s_endpgm
7223;
7224; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7225; GFX10-CU:       ; %bb.0: ; %entry
7226; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7227; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7228; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7229; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7230; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7231; GFX10-CU-NEXT:    s_endpgm
7232;
7233; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7234; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7235; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7236; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7237; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7238; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7239; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7240; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7241; SKIP-CACHE-INV-NEXT:    s_endpgm
7242;
7243; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7244; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7245; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7246; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7247; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7248; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7249; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7250; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7251;
7252; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7253; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7254; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7255; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7256; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7257; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7258; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7259; GFX90A-TGSPLIT-NEXT:    s_endpgm
7260;
7261; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7262; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7263; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7264; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7265; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7266; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7267; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7268; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7269;
7270; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7271; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7272; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7273; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7274; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7275; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7276; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
7277; GFX940-TGSPLIT-NEXT:    s_endpgm
7278;
7279; GFX11-WGP-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7280; GFX11-WGP:       ; %bb.0: ; %entry
7281; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7282; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7283; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7284; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
7285; GFX11-WGP-NEXT:    s_endpgm
7286;
7287; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
7288; GFX11-CU:       ; %bb.0: ; %entry
7289; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7290; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7291; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7292; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
7293; GFX11-CU-NEXT:    s_endpgm
7294    i32 addrspace(3)* %out, i32 %in) {
7295entry:
7296  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") seq_cst
7297  ret void
7298}
7299
7300define amdgpu_kernel void @local_singlethread_one_as_acquire_ret_atomicrmw(
7301; GFX6-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7302; GFX6:       ; %bb.0: ; %entry
7303; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
7304; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
7305; GFX6-NEXT:    s_mov_b32 m0, -1
7306; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7307; GFX6-NEXT:    v_mov_b32_e32 v0, s0
7308; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7309; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7310; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7311; GFX6-NEXT:    ds_write_b32 v0, v1
7312; GFX6-NEXT:    s_endpgm
7313;
7314; GFX7-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7315; GFX7:       ; %bb.0: ; %entry
7316; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7317; GFX7-NEXT:    s_mov_b32 m0, -1
7318; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7319; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7320; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7321; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7322; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7323; GFX7-NEXT:    ds_write_b32 v0, v1
7324; GFX7-NEXT:    s_endpgm
7325;
7326; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7327; GFX10-WGP:       ; %bb.0: ; %entry
7328; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7329; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7330; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7331; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7332; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7333; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7334; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
7335; GFX10-WGP-NEXT:    s_endpgm
7336;
7337; GFX10-CU-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7338; GFX10-CU:       ; %bb.0: ; %entry
7339; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7340; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7341; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7342; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7343; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7344; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7345; GFX10-CU-NEXT:    ds_write_b32 v0, v1
7346; GFX10-CU-NEXT:    s_endpgm
7347;
7348; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7349; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7350; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7351; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7352; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7353; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7354; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7355; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7356; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7357; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
7358; SKIP-CACHE-INV-NEXT:    s_endpgm
7359;
7360; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7361; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7362; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7363; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7364; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7365; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7366; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7367; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7368; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7369; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7370;
7371; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7372; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7373; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7374; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7375; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7376; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7377; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7378; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7379; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7380; GFX90A-TGSPLIT-NEXT:    s_endpgm
7381;
7382; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7383; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7384; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7385; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7386; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7387; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7388; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7389; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7390; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7391; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7392;
7393; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7394; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7395; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7396; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7397; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7398; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7399; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7400; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7401; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7402; GFX940-TGSPLIT-NEXT:    s_endpgm
7403;
7404; GFX11-WGP-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7405; GFX11-WGP:       ; %bb.0: ; %entry
7406; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7407; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7408; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7409; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
7410; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7411; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
7412; GFX11-WGP-NEXT:    s_endpgm
7413;
7414; GFX11-CU-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
7415; GFX11-CU:       ; %bb.0: ; %entry
7416; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7417; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7418; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7419; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
7420; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7421; GFX11-CU-NEXT:    ds_store_b32 v0, v1
7422; GFX11-CU-NEXT:    s_endpgm
7423    i32 addrspace(3)* %out, i32 %in) {
7424entry:
7425  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") acquire
7426  store i32 %val, i32 addrspace(3)* %out, align 4
7427  ret void
7428}
7429
7430define amdgpu_kernel void @local_singlethread_one_as_acq_rel_ret_atomicrmw(
7431; GFX6-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7432; GFX6:       ; %bb.0: ; %entry
7433; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
7434; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
7435; GFX6-NEXT:    s_mov_b32 m0, -1
7436; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7437; GFX6-NEXT:    v_mov_b32_e32 v0, s0
7438; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7439; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7440; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7441; GFX6-NEXT:    ds_write_b32 v0, v1
7442; GFX6-NEXT:    s_endpgm
7443;
7444; GFX7-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7445; GFX7:       ; %bb.0: ; %entry
7446; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7447; GFX7-NEXT:    s_mov_b32 m0, -1
7448; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7449; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7450; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7451; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7452; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7453; GFX7-NEXT:    ds_write_b32 v0, v1
7454; GFX7-NEXT:    s_endpgm
7455;
7456; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7457; GFX10-WGP:       ; %bb.0: ; %entry
7458; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7459; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7460; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7461; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7462; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7463; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7464; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
7465; GFX10-WGP-NEXT:    s_endpgm
7466;
7467; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7468; GFX10-CU:       ; %bb.0: ; %entry
7469; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7470; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7471; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7472; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7473; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7474; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7475; GFX10-CU-NEXT:    ds_write_b32 v0, v1
7476; GFX10-CU-NEXT:    s_endpgm
7477;
7478; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7479; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7480; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7481; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7482; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7483; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7484; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7485; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7486; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7487; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
7488; SKIP-CACHE-INV-NEXT:    s_endpgm
7489;
7490; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7491; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7492; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7493; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7494; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7495; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7496; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7497; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7498; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7499; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7500;
7501; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7502; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7503; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7504; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7505; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7506; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7507; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7508; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7509; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7510; GFX90A-TGSPLIT-NEXT:    s_endpgm
7511;
7512; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7513; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7514; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7515; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7516; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7517; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7518; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7519; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7520; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7521; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7522;
7523; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7524; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7525; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7526; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7527; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7528; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7529; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7530; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7531; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7532; GFX940-TGSPLIT-NEXT:    s_endpgm
7533;
7534; GFX11-WGP-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7535; GFX11-WGP:       ; %bb.0: ; %entry
7536; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7537; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7538; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7539; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
7540; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7541; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
7542; GFX11-WGP-NEXT:    s_endpgm
7543;
7544; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
7545; GFX11-CU:       ; %bb.0: ; %entry
7546; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7547; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7548; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7549; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
7550; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7551; GFX11-CU-NEXT:    ds_store_b32 v0, v1
7552; GFX11-CU-NEXT:    s_endpgm
7553    i32 addrspace(3)* %out, i32 %in) {
7554entry:
7555  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") acq_rel
7556  store i32 %val, i32 addrspace(3)* %out, align 4
7557  ret void
7558}
7559
7560define amdgpu_kernel void @local_singlethread_one_as_seq_cst_ret_atomicrmw(
7561; GFX6-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7562; GFX6:       ; %bb.0: ; %entry
7563; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
7564; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
7565; GFX6-NEXT:    s_mov_b32 m0, -1
7566; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7567; GFX6-NEXT:    v_mov_b32_e32 v0, s0
7568; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7569; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7570; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7571; GFX6-NEXT:    ds_write_b32 v0, v1
7572; GFX6-NEXT:    s_endpgm
7573;
7574; GFX7-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7575; GFX7:       ; %bb.0: ; %entry
7576; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7577; GFX7-NEXT:    s_mov_b32 m0, -1
7578; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7579; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7580; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7581; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7582; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7583; GFX7-NEXT:    ds_write_b32 v0, v1
7584; GFX7-NEXT:    s_endpgm
7585;
7586; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7587; GFX10-WGP:       ; %bb.0: ; %entry
7588; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7589; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7590; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7591; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7592; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7593; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7594; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
7595; GFX10-WGP-NEXT:    s_endpgm
7596;
7597; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7598; GFX10-CU:       ; %bb.0: ; %entry
7599; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7600; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7601; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7602; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7603; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7604; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7605; GFX10-CU-NEXT:    ds_write_b32 v0, v1
7606; GFX10-CU-NEXT:    s_endpgm
7607;
7608; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7609; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7610; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7611; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7612; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7613; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7614; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7615; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7616; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7617; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
7618; SKIP-CACHE-INV-NEXT:    s_endpgm
7619;
7620; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7621; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7622; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7623; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7624; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7625; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7626; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7627; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7628; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7629; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7630;
7631; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7632; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7633; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7634; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7635; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7636; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7637; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7638; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7639; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7640; GFX90A-TGSPLIT-NEXT:    s_endpgm
7641;
7642; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7643; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7644; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7645; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7646; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7647; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7648; GFX940-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7649; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7650; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7651; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7652;
7653; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7654; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7655; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7656; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7657; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7658; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7659; GFX940-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
7660; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7661; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7662; GFX940-TGSPLIT-NEXT:    s_endpgm
7663;
7664; GFX11-WGP-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7665; GFX11-WGP:       ; %bb.0: ; %entry
7666; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7667; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7668; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7669; GFX11-WGP-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
7670; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7671; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
7672; GFX11-WGP-NEXT:    s_endpgm
7673;
7674; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
7675; GFX11-CU:       ; %bb.0: ; %entry
7676; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7677; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7678; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7679; GFX11-CU-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
7680; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7681; GFX11-CU-NEXT:    ds_store_b32 v0, v1
7682; GFX11-CU-NEXT:    s_endpgm
7683    i32 addrspace(3)* %out, i32 %in) {
7684entry:
7685  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") seq_cst
7686  store i32 %val, i32 addrspace(3)* %out, align 4
7687  ret void
7688}
7689
7690define amdgpu_kernel void @local_singlethread_one_as_monotonic_monotonic_cmpxchg(
7691; GFX6-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7692; GFX6:       ; %bb.0: ; %entry
7693; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
7694; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
7695; GFX6-NEXT:    s_mov_b32 m0, -1
7696; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7697; GFX6-NEXT:    v_mov_b32_e32 v0, s2
7698; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7699; GFX6-NEXT:    v_mov_b32_e32 v2, s0
7700; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7701; GFX6-NEXT:    s_endpgm
7702;
7703; GFX7-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7704; GFX7:       ; %bb.0: ; %entry
7705; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7706; GFX7-NEXT:    s_mov_b32 m0, -1
7707; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7708; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7709; GFX7-NEXT:    v_mov_b32_e32 v1, s2
7710; GFX7-NEXT:    v_mov_b32_e32 v2, s1
7711; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7712; GFX7-NEXT:    s_endpgm
7713;
7714; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7715; GFX10-WGP:       ; %bb.0: ; %entry
7716; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7717; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7718; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7719; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7720; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
7721; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7722; GFX10-WGP-NEXT:    s_endpgm
7723;
7724; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7725; GFX10-CU:       ; %bb.0: ; %entry
7726; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7727; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7728; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7729; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7730; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
7731; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7732; GFX10-CU-NEXT:    s_endpgm
7733;
7734; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7735; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7736; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7737; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7738; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7739; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7740; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
7741; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
7742; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7743; SKIP-CACHE-INV-NEXT:    s_endpgm
7744;
7745; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7746; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7747; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7748; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7749; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7750; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7751; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7752; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7753; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7754;
7755; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7756; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7757; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7758; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7759; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7760; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7761; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7762; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7763; GFX90A-TGSPLIT-NEXT:    s_endpgm
7764;
7765; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7766; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7767; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7768; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7769; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7770; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7771; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7772; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7773; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7774;
7775; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7776; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7777; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7778; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7779; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7780; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7781; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7782; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7783; GFX940-TGSPLIT-NEXT:    s_endpgm
7784;
7785; GFX11-WGP-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7786; GFX11-WGP:       ; %bb.0: ; %entry
7787; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7788; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7789; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7790; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
7791; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
7792; GFX11-WGP-NEXT:    s_endpgm
7793;
7794; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
7795; GFX11-CU:       ; %bb.0: ; %entry
7796; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7797; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7798; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7799; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
7800; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
7801; GFX11-CU-NEXT:    s_endpgm
7802    i32 addrspace(3)* %out, i32 %in, i32 %old) {
7803entry:
7804  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
7805  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
7806  ret void
7807}
7808
7809define amdgpu_kernel void @local_singlethread_one_as_acquire_monotonic_cmpxchg(
7810; GFX6-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
7811; GFX6:       ; %bb.0: ; %entry
7812; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
7813; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
7814; GFX6-NEXT:    s_mov_b32 m0, -1
7815; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7816; GFX6-NEXT:    v_mov_b32_e32 v0, s2
7817; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7818; GFX6-NEXT:    v_mov_b32_e32 v2, s0
7819; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7820; GFX6-NEXT:    s_endpgm
7821;
7822; GFX7-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
7823; GFX7:       ; %bb.0: ; %entry
7824; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7825; GFX7-NEXT:    s_mov_b32 m0, -1
7826; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7827; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7828; GFX7-NEXT:    v_mov_b32_e32 v1, s2
7829; GFX7-NEXT:    v_mov_b32_e32 v2, s1
7830; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7831; GFX7-NEXT:    s_endpgm
7832;
7833; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
7834; GFX10-WGP:       ; %bb.0: ; %entry
7835; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7836; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7837; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7838; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7839; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
7840; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7841; GFX10-WGP-NEXT:    s_endpgm
7842;
7843; GFX10-CU-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
7844; GFX10-CU:       ; %bb.0: ; %entry
7845; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7846; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7847; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7848; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7849; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
7850; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7851; GFX10-CU-NEXT:    s_endpgm
7852;
7853; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
7854; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7855; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7856; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7857; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7858; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7859; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
7860; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
7861; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7862; SKIP-CACHE-INV-NEXT:    s_endpgm
7863;
7864; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
7865; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7866; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7867; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7868; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7869; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7870; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7871; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7872; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7873;
7874; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
7875; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7876; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7877; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7878; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7879; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7880; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7881; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7882; GFX90A-TGSPLIT-NEXT:    s_endpgm
7883;
7884; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
7885; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7886; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7887; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7888; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7889; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7890; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7891; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7892; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7893;
7894; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
7895; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7896; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7897; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7898; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7899; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7900; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7901; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7902; GFX940-TGSPLIT-NEXT:    s_endpgm
7903;
7904; GFX11-WGP-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
7905; GFX11-WGP:       ; %bb.0: ; %entry
7906; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7907; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7908; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7909; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
7910; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
7911; GFX11-WGP-NEXT:    s_endpgm
7912;
7913; GFX11-CU-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
7914; GFX11-CU:       ; %bb.0: ; %entry
7915; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7916; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7917; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7918; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
7919; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
7920; GFX11-CU-NEXT:    s_endpgm
7921    i32 addrspace(3)* %out, i32 %in, i32 %old) {
7922entry:
7923  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
7924  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
7925  ret void
7926}
7927
7928define amdgpu_kernel void @local_singlethread_one_as_release_monotonic_cmpxchg(
7929; GFX6-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
7930; GFX6:       ; %bb.0: ; %entry
7931; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
7932; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
7933; GFX6-NEXT:    s_mov_b32 m0, -1
7934; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7935; GFX6-NEXT:    v_mov_b32_e32 v0, s2
7936; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7937; GFX6-NEXT:    v_mov_b32_e32 v2, s0
7938; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7939; GFX6-NEXT:    s_endpgm
7940;
7941; GFX7-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
7942; GFX7:       ; %bb.0: ; %entry
7943; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7944; GFX7-NEXT:    s_mov_b32 m0, -1
7945; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7946; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7947; GFX7-NEXT:    v_mov_b32_e32 v1, s2
7948; GFX7-NEXT:    v_mov_b32_e32 v2, s1
7949; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7950; GFX7-NEXT:    s_endpgm
7951;
7952; GFX10-WGP-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
7953; GFX10-WGP:       ; %bb.0: ; %entry
7954; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7955; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7956; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7957; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7958; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
7959; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7960; GFX10-WGP-NEXT:    s_endpgm
7961;
7962; GFX10-CU-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
7963; GFX10-CU:       ; %bb.0: ; %entry
7964; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7965; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7966; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7967; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7968; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
7969; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7970; GFX10-CU-NEXT:    s_endpgm
7971;
7972; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
7973; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7974; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7975; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7976; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7977; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7978; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
7979; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
7980; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7981; SKIP-CACHE-INV-NEXT:    s_endpgm
7982;
7983; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
7984; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7985; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7986; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7987; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7988; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7989; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7990; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
7991; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7992;
7993; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
7994; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7995; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7996; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7997; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7998; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7999; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8000; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8001; GFX90A-TGSPLIT-NEXT:    s_endpgm
8002;
8003; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
8004; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8005; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8006; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8007; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8008; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8009; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8010; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8011; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8012;
8013; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
8014; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8015; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8016; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8017; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8018; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8019; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8020; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8021; GFX940-TGSPLIT-NEXT:    s_endpgm
8022;
8023; GFX11-WGP-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
8024; GFX11-WGP:       ; %bb.0: ; %entry
8025; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8026; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8027; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8028; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
8029; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8030; GFX11-WGP-NEXT:    s_endpgm
8031;
8032; GFX11-CU-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
8033; GFX11-CU:       ; %bb.0: ; %entry
8034; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8035; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8036; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8037; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8038; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8039; GFX11-CU-NEXT:    s_endpgm
8040    i32 addrspace(3)* %out, i32 %in, i32 %old) {
8041entry:
8042  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
8043  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
8044  ret void
8045}
8046
8047define amdgpu_kernel void @local_singlethread_one_as_acq_rel_monotonic_cmpxchg(
8048; GFX6-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8049; GFX6:       ; %bb.0: ; %entry
8050; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
8051; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
8052; GFX6-NEXT:    s_mov_b32 m0, -1
8053; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8054; GFX6-NEXT:    v_mov_b32_e32 v0, s2
8055; GFX6-NEXT:    v_mov_b32_e32 v1, s1
8056; GFX6-NEXT:    v_mov_b32_e32 v2, s0
8057; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8058; GFX6-NEXT:    s_endpgm
8059;
8060; GFX7-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8061; GFX7:       ; %bb.0: ; %entry
8062; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8063; GFX7-NEXT:    s_mov_b32 m0, -1
8064; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8065; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8066; GFX7-NEXT:    v_mov_b32_e32 v1, s2
8067; GFX7-NEXT:    v_mov_b32_e32 v2, s1
8068; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8069; GFX7-NEXT:    s_endpgm
8070;
8071; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8072; GFX10-WGP:       ; %bb.0: ; %entry
8073; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8074; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8075; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8076; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8077; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
8078; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8079; GFX10-WGP-NEXT:    s_endpgm
8080;
8081; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8082; GFX10-CU:       ; %bb.0: ; %entry
8083; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8084; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8085; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8086; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8087; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8088; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8089; GFX10-CU-NEXT:    s_endpgm
8090;
8091; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8092; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8093; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8094; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
8095; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8096; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8097; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
8098; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
8099; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8100; SKIP-CACHE-INV-NEXT:    s_endpgm
8101;
8102; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8103; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8104; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8105; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8106; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8107; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8108; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8109; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8110; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8111;
8112; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8113; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8114; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8115; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8116; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8117; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8118; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8119; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8120; GFX90A-TGSPLIT-NEXT:    s_endpgm
8121;
8122; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8123; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8124; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8125; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8126; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8127; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8128; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8129; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8130; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8131;
8132; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8133; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8134; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8135; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8136; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8137; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8138; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8139; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8140; GFX940-TGSPLIT-NEXT:    s_endpgm
8141;
8142; GFX11-WGP-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8143; GFX11-WGP:       ; %bb.0: ; %entry
8144; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8145; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8146; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8147; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
8148; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8149; GFX11-WGP-NEXT:    s_endpgm
8150;
8151; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8152; GFX11-CU:       ; %bb.0: ; %entry
8153; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8154; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8155; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8156; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8157; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8158; GFX11-CU-NEXT:    s_endpgm
8159    i32 addrspace(3)* %out, i32 %in, i32 %old) {
8160entry:
8161  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
8162  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
8163  ret void
8164}
8165
8166define amdgpu_kernel void @local_singlethread_one_as_seq_cst_monotonic_cmpxchg(
8167; GFX6-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8168; GFX6:       ; %bb.0: ; %entry
8169; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
8170; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
8171; GFX6-NEXT:    s_mov_b32 m0, -1
8172; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8173; GFX6-NEXT:    v_mov_b32_e32 v0, s2
8174; GFX6-NEXT:    v_mov_b32_e32 v1, s1
8175; GFX6-NEXT:    v_mov_b32_e32 v2, s0
8176; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8177; GFX6-NEXT:    s_endpgm
8178;
8179; GFX7-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8180; GFX7:       ; %bb.0: ; %entry
8181; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8182; GFX7-NEXT:    s_mov_b32 m0, -1
8183; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8184; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8185; GFX7-NEXT:    v_mov_b32_e32 v1, s2
8186; GFX7-NEXT:    v_mov_b32_e32 v2, s1
8187; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8188; GFX7-NEXT:    s_endpgm
8189;
8190; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8191; GFX10-WGP:       ; %bb.0: ; %entry
8192; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8193; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8194; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8195; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8196; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
8197; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8198; GFX10-WGP-NEXT:    s_endpgm
8199;
8200; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8201; GFX10-CU:       ; %bb.0: ; %entry
8202; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8203; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8204; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8205; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8206; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8207; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8208; GFX10-CU-NEXT:    s_endpgm
8209;
8210; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8211; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8212; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8213; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
8214; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8215; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8216; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
8217; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
8218; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8219; SKIP-CACHE-INV-NEXT:    s_endpgm
8220;
8221; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8222; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8223; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8224; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8225; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8226; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8227; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8228; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8229; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8230;
8231; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8232; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8233; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8234; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8235; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8236; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8237; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8238; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8239; GFX90A-TGSPLIT-NEXT:    s_endpgm
8240;
8241; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8242; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8243; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8244; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8245; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8246; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8247; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8248; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8249; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8250;
8251; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8252; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8253; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8254; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8255; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8256; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8257; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8258; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8259; GFX940-TGSPLIT-NEXT:    s_endpgm
8260;
8261; GFX11-WGP-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8262; GFX11-WGP:       ; %bb.0: ; %entry
8263; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8264; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8265; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8266; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
8267; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8268; GFX11-WGP-NEXT:    s_endpgm
8269;
8270; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
8271; GFX11-CU:       ; %bb.0: ; %entry
8272; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8273; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8274; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8275; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8276; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8277; GFX11-CU-NEXT:    s_endpgm
8278    i32 addrspace(3)* %out, i32 %in, i32 %old) {
8279entry:
8280  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
8281  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
8282  ret void
8283}
8284
8285define amdgpu_kernel void @local_singlethread_one_as_monotonic_acquire_cmpxchg(
8286; GFX6-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8287; GFX6:       ; %bb.0: ; %entry
8288; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
8289; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
8290; GFX6-NEXT:    s_mov_b32 m0, -1
8291; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8292; GFX6-NEXT:    v_mov_b32_e32 v0, s2
8293; GFX6-NEXT:    v_mov_b32_e32 v1, s1
8294; GFX6-NEXT:    v_mov_b32_e32 v2, s0
8295; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8296; GFX6-NEXT:    s_endpgm
8297;
8298; GFX7-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8299; GFX7:       ; %bb.0: ; %entry
8300; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8301; GFX7-NEXT:    s_mov_b32 m0, -1
8302; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8303; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8304; GFX7-NEXT:    v_mov_b32_e32 v1, s2
8305; GFX7-NEXT:    v_mov_b32_e32 v2, s1
8306; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8307; GFX7-NEXT:    s_endpgm
8308;
8309; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8310; GFX10-WGP:       ; %bb.0: ; %entry
8311; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8312; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8313; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8314; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8315; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
8316; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8317; GFX10-WGP-NEXT:    s_endpgm
8318;
8319; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8320; GFX10-CU:       ; %bb.0: ; %entry
8321; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8322; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8323; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8324; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8325; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8326; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8327; GFX10-CU-NEXT:    s_endpgm
8328;
8329; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8330; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8331; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8332; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
8333; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8334; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8335; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
8336; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
8337; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8338; SKIP-CACHE-INV-NEXT:    s_endpgm
8339;
8340; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8341; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8342; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8343; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8344; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8345; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8346; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8347; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8348; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8349;
8350; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8351; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8352; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8353; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8354; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8355; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8356; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8357; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8358; GFX90A-TGSPLIT-NEXT:    s_endpgm
8359;
8360; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8361; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8362; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8363; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8364; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8365; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8366; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8367; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8368; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8369;
8370; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8371; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8372; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8373; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8374; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8375; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8376; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8377; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8378; GFX940-TGSPLIT-NEXT:    s_endpgm
8379;
8380; GFX11-WGP-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8381; GFX11-WGP:       ; %bb.0: ; %entry
8382; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8383; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8384; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8385; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
8386; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8387; GFX11-WGP-NEXT:    s_endpgm
8388;
8389; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
8390; GFX11-CU:       ; %bb.0: ; %entry
8391; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8392; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8393; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8394; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8395; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8396; GFX11-CU-NEXT:    s_endpgm
8397    i32 addrspace(3)* %out, i32 %in, i32 %old) {
8398entry:
8399  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
8400  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic acquire
8401  ret void
8402}
8403
8404define amdgpu_kernel void @local_singlethread_one_as_acquire_acquire_cmpxchg(
8405; GFX6-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8406; GFX6:       ; %bb.0: ; %entry
8407; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
8408; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
8409; GFX6-NEXT:    s_mov_b32 m0, -1
8410; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8411; GFX6-NEXT:    v_mov_b32_e32 v0, s2
8412; GFX6-NEXT:    v_mov_b32_e32 v1, s1
8413; GFX6-NEXT:    v_mov_b32_e32 v2, s0
8414; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8415; GFX6-NEXT:    s_endpgm
8416;
8417; GFX7-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8418; GFX7:       ; %bb.0: ; %entry
8419; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8420; GFX7-NEXT:    s_mov_b32 m0, -1
8421; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8422; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8423; GFX7-NEXT:    v_mov_b32_e32 v1, s2
8424; GFX7-NEXT:    v_mov_b32_e32 v2, s1
8425; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8426; GFX7-NEXT:    s_endpgm
8427;
8428; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8429; GFX10-WGP:       ; %bb.0: ; %entry
8430; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8431; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8432; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8433; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8434; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
8435; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8436; GFX10-WGP-NEXT:    s_endpgm
8437;
8438; GFX10-CU-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8439; GFX10-CU:       ; %bb.0: ; %entry
8440; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8441; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8442; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8443; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8444; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8445; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8446; GFX10-CU-NEXT:    s_endpgm
8447;
8448; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8449; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8450; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8451; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
8452; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8453; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8454; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
8455; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
8456; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8457; SKIP-CACHE-INV-NEXT:    s_endpgm
8458;
8459; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8460; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8461; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8462; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8463; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8464; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8465; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8466; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8467; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8468;
8469; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8470; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8471; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8472; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8473; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8474; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8475; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8476; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8477; GFX90A-TGSPLIT-NEXT:    s_endpgm
8478;
8479; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8480; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8481; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8482; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8483; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8484; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8485; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8486; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8487; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8488;
8489; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8490; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8491; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8492; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8493; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8494; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8495; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8496; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8497; GFX940-TGSPLIT-NEXT:    s_endpgm
8498;
8499; GFX11-WGP-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8500; GFX11-WGP:       ; %bb.0: ; %entry
8501; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8502; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8503; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8504; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
8505; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8506; GFX11-WGP-NEXT:    s_endpgm
8507;
8508; GFX11-CU-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
8509; GFX11-CU:       ; %bb.0: ; %entry
8510; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8511; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8512; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8513; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8514; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8515; GFX11-CU-NEXT:    s_endpgm
8516    i32 addrspace(3)* %out, i32 %in, i32 %old) {
8517entry:
8518  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
8519  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
8520  ret void
8521}
8522
8523define amdgpu_kernel void @local_singlethread_one_as_release_acquire_cmpxchg(
8524; GFX6-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8525; GFX6:       ; %bb.0: ; %entry
8526; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
8527; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
8528; GFX6-NEXT:    s_mov_b32 m0, -1
8529; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8530; GFX6-NEXT:    v_mov_b32_e32 v0, s2
8531; GFX6-NEXT:    v_mov_b32_e32 v1, s1
8532; GFX6-NEXT:    v_mov_b32_e32 v2, s0
8533; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8534; GFX6-NEXT:    s_endpgm
8535;
8536; GFX7-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8537; GFX7:       ; %bb.0: ; %entry
8538; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8539; GFX7-NEXT:    s_mov_b32 m0, -1
8540; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8541; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8542; GFX7-NEXT:    v_mov_b32_e32 v1, s2
8543; GFX7-NEXT:    v_mov_b32_e32 v2, s1
8544; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8545; GFX7-NEXT:    s_endpgm
8546;
8547; GFX10-WGP-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8548; GFX10-WGP:       ; %bb.0: ; %entry
8549; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8550; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8551; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8552; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8553; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
8554; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8555; GFX10-WGP-NEXT:    s_endpgm
8556;
8557; GFX10-CU-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8558; GFX10-CU:       ; %bb.0: ; %entry
8559; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8560; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8561; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8562; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8563; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8564; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8565; GFX10-CU-NEXT:    s_endpgm
8566;
8567; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8568; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8569; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8570; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
8571; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8572; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8573; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
8574; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
8575; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8576; SKIP-CACHE-INV-NEXT:    s_endpgm
8577;
8578; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8579; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8580; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8581; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8582; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8583; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8584; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8585; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8586; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8587;
8588; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8589; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8590; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8591; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8592; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8593; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8594; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8595; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8596; GFX90A-TGSPLIT-NEXT:    s_endpgm
8597;
8598; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8599; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8600; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8601; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8602; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8603; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8604; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8605; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8606; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8607;
8608; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8609; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8610; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8611; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8612; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8613; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8614; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8615; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8616; GFX940-TGSPLIT-NEXT:    s_endpgm
8617;
8618; GFX11-WGP-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8619; GFX11-WGP:       ; %bb.0: ; %entry
8620; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8621; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8622; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8623; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
8624; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8625; GFX11-WGP-NEXT:    s_endpgm
8626;
8627; GFX11-CU-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
8628; GFX11-CU:       ; %bb.0: ; %entry
8629; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8630; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8631; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8632; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8633; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8634; GFX11-CU-NEXT:    s_endpgm
8635    i32 addrspace(3)* %out, i32 %in, i32 %old) {
8636entry:
8637  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
8638  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
8639  ret void
8640}
8641
8642define amdgpu_kernel void @local_singlethread_one_as_acq_rel_acquire_cmpxchg(
8643; GFX6-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8644; GFX6:       ; %bb.0: ; %entry
8645; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
8646; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
8647; GFX6-NEXT:    s_mov_b32 m0, -1
8648; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8649; GFX6-NEXT:    v_mov_b32_e32 v0, s2
8650; GFX6-NEXT:    v_mov_b32_e32 v1, s1
8651; GFX6-NEXT:    v_mov_b32_e32 v2, s0
8652; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8653; GFX6-NEXT:    s_endpgm
8654;
8655; GFX7-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8656; GFX7:       ; %bb.0: ; %entry
8657; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8658; GFX7-NEXT:    s_mov_b32 m0, -1
8659; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8660; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8661; GFX7-NEXT:    v_mov_b32_e32 v1, s2
8662; GFX7-NEXT:    v_mov_b32_e32 v2, s1
8663; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8664; GFX7-NEXT:    s_endpgm
8665;
8666; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8667; GFX10-WGP:       ; %bb.0: ; %entry
8668; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8669; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8670; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8671; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8672; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
8673; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8674; GFX10-WGP-NEXT:    s_endpgm
8675;
8676; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8677; GFX10-CU:       ; %bb.0: ; %entry
8678; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8679; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8680; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8681; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8682; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8683; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8684; GFX10-CU-NEXT:    s_endpgm
8685;
8686; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8687; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8688; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8689; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
8690; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8691; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8692; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
8693; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
8694; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8695; SKIP-CACHE-INV-NEXT:    s_endpgm
8696;
8697; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8698; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8699; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8700; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8701; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8702; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8703; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8704; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8705; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8706;
8707; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8708; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8709; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8710; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8711; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8712; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8713; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8714; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8715; GFX90A-TGSPLIT-NEXT:    s_endpgm
8716;
8717; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8718; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8719; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8720; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8721; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8722; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8723; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8724; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8725; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8726;
8727; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8728; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8729; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8730; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8731; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8732; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8733; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8734; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8735; GFX940-TGSPLIT-NEXT:    s_endpgm
8736;
8737; GFX11-WGP-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8738; GFX11-WGP:       ; %bb.0: ; %entry
8739; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8740; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8741; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8742; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
8743; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8744; GFX11-WGP-NEXT:    s_endpgm
8745;
8746; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
8747; GFX11-CU:       ; %bb.0: ; %entry
8748; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8749; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8750; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8751; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8752; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8753; GFX11-CU-NEXT:    s_endpgm
8754    i32 addrspace(3)* %out, i32 %in, i32 %old) {
8755entry:
8756  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
8757  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
8758  ret void
8759}
8760
8761define amdgpu_kernel void @local_singlethread_one_as_seq_cst_acquire_cmpxchg(
8762; GFX6-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8763; GFX6:       ; %bb.0: ; %entry
8764; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
8765; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
8766; GFX6-NEXT:    s_mov_b32 m0, -1
8767; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8768; GFX6-NEXT:    v_mov_b32_e32 v0, s2
8769; GFX6-NEXT:    v_mov_b32_e32 v1, s1
8770; GFX6-NEXT:    v_mov_b32_e32 v2, s0
8771; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8772; GFX6-NEXT:    s_endpgm
8773;
8774; GFX7-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8775; GFX7:       ; %bb.0: ; %entry
8776; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8777; GFX7-NEXT:    s_mov_b32 m0, -1
8778; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8779; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8780; GFX7-NEXT:    v_mov_b32_e32 v1, s2
8781; GFX7-NEXT:    v_mov_b32_e32 v2, s1
8782; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8783; GFX7-NEXT:    s_endpgm
8784;
8785; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8786; GFX10-WGP:       ; %bb.0: ; %entry
8787; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8788; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8789; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8790; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8791; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
8792; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8793; GFX10-WGP-NEXT:    s_endpgm
8794;
8795; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8796; GFX10-CU:       ; %bb.0: ; %entry
8797; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8798; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8799; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8800; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8801; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8802; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8803; GFX10-CU-NEXT:    s_endpgm
8804;
8805; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8806; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8807; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8808; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
8809; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8810; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8811; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
8812; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
8813; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8814; SKIP-CACHE-INV-NEXT:    s_endpgm
8815;
8816; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8817; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8818; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8819; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8820; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8821; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8822; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8823; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8824; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8825;
8826; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8827; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8828; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8829; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8830; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8831; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8832; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8833; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8834; GFX90A-TGSPLIT-NEXT:    s_endpgm
8835;
8836; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8837; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8838; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8839; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8840; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8841; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8842; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8843; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8844; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8845;
8846; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8847; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8848; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8849; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8850; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8851; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8852; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8853; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8854; GFX940-TGSPLIT-NEXT:    s_endpgm
8855;
8856; GFX11-WGP-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8857; GFX11-WGP:       ; %bb.0: ; %entry
8858; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8859; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8860; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8861; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
8862; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8863; GFX11-WGP-NEXT:    s_endpgm
8864;
8865; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
8866; GFX11-CU:       ; %bb.0: ; %entry
8867; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8868; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8869; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8870; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8871; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8872; GFX11-CU-NEXT:    s_endpgm
8873    i32 addrspace(3)* %out, i32 %in, i32 %old) {
8874entry:
8875  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
8876  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
8877  ret void
8878}
8879
8880define amdgpu_kernel void @local_singlethread_one_as_monotonic_seq_cst_cmpxchg(
8881; GFX6-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8882; GFX6:       ; %bb.0: ; %entry
8883; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
8884; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
8885; GFX6-NEXT:    s_mov_b32 m0, -1
8886; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8887; GFX6-NEXT:    v_mov_b32_e32 v0, s2
8888; GFX6-NEXT:    v_mov_b32_e32 v1, s1
8889; GFX6-NEXT:    v_mov_b32_e32 v2, s0
8890; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8891; GFX6-NEXT:    s_endpgm
8892;
8893; GFX7-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8894; GFX7:       ; %bb.0: ; %entry
8895; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8896; GFX7-NEXT:    s_mov_b32 m0, -1
8897; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8898; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8899; GFX7-NEXT:    v_mov_b32_e32 v1, s2
8900; GFX7-NEXT:    v_mov_b32_e32 v2, s1
8901; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8902; GFX7-NEXT:    s_endpgm
8903;
8904; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8905; GFX10-WGP:       ; %bb.0: ; %entry
8906; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8907; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8908; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8909; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8910; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
8911; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8912; GFX10-WGP-NEXT:    s_endpgm
8913;
8914; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8915; GFX10-CU:       ; %bb.0: ; %entry
8916; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8917; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8918; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8919; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8920; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
8921; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8922; GFX10-CU-NEXT:    s_endpgm
8923;
8924; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8925; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8926; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8927; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
8928; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8929; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8930; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
8931; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
8932; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8933; SKIP-CACHE-INV-NEXT:    s_endpgm
8934;
8935; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8936; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8937; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8938; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8939; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8940; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8941; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8942; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8943; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8944;
8945; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8946; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8947; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8948; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8949; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8950; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8951; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8952; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8953; GFX90A-TGSPLIT-NEXT:    s_endpgm
8954;
8955; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8956; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8957; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8958; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8959; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8960; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8961; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8962; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8963; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8964;
8965; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8966; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8967; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8968; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8969; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8970; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8971; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
8972; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
8973; GFX940-TGSPLIT-NEXT:    s_endpgm
8974;
8975; GFX11-WGP-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8976; GFX11-WGP:       ; %bb.0: ; %entry
8977; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8978; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8979; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8980; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
8981; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8982; GFX11-WGP-NEXT:    s_endpgm
8983;
8984; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8985; GFX11-CU:       ; %bb.0: ; %entry
8986; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8987; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8988; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8989; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
8990; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
8991; GFX11-CU-NEXT:    s_endpgm
8992    i32 addrspace(3)* %out, i32 %in, i32 %old) {
8993entry:
8994  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
8995  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic seq_cst
8996  ret void
8997}
8998
8999define amdgpu_kernel void @local_singlethread_one_as_acquire_seq_cst_cmpxchg(
9000; GFX6-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9001; GFX6:       ; %bb.0: ; %entry
9002; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
9003; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
9004; GFX6-NEXT:    s_mov_b32 m0, -1
9005; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9006; GFX6-NEXT:    v_mov_b32_e32 v0, s2
9007; GFX6-NEXT:    v_mov_b32_e32 v1, s1
9008; GFX6-NEXT:    v_mov_b32_e32 v2, s0
9009; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9010; GFX6-NEXT:    s_endpgm
9011;
9012; GFX7-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9013; GFX7:       ; %bb.0: ; %entry
9014; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9015; GFX7-NEXT:    s_mov_b32 m0, -1
9016; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9017; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9018; GFX7-NEXT:    v_mov_b32_e32 v1, s2
9019; GFX7-NEXT:    v_mov_b32_e32 v2, s1
9020; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9021; GFX7-NEXT:    s_endpgm
9022;
9023; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9024; GFX10-WGP:       ; %bb.0: ; %entry
9025; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9026; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9027; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9028; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
9029; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
9030; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9031; GFX10-WGP-NEXT:    s_endpgm
9032;
9033; GFX10-CU-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9034; GFX10-CU:       ; %bb.0: ; %entry
9035; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9036; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9037; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9038; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9039; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9040; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9041; GFX10-CU-NEXT:    s_endpgm
9042;
9043; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9044; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9045; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9046; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
9047; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9048; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9049; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
9050; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
9051; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9052; SKIP-CACHE-INV-NEXT:    s_endpgm
9053;
9054; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9055; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9056; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9057; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9058; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9059; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9060; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9061; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9062; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9063;
9064; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9065; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9066; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9067; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9068; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9069; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9070; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9071; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9072; GFX90A-TGSPLIT-NEXT:    s_endpgm
9073;
9074; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9075; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9076; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9077; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9078; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9079; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9080; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9081; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9082; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9083;
9084; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9085; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9086; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9087; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9088; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9089; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9090; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9091; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9092; GFX940-TGSPLIT-NEXT:    s_endpgm
9093;
9094; GFX11-WGP-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9095; GFX11-WGP:       ; %bb.0: ; %entry
9096; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9097; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9098; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9099; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
9100; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9101; GFX11-WGP-NEXT:    s_endpgm
9102;
9103; GFX11-CU-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
9104; GFX11-CU:       ; %bb.0: ; %entry
9105; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9106; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9107; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9108; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9109; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9110; GFX11-CU-NEXT:    s_endpgm
9111    i32 addrspace(3)* %out, i32 %in, i32 %old) {
9112entry:
9113  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
9114  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire seq_cst
9115  ret void
9116}
9117
9118define amdgpu_kernel void @local_singlethread_one_as_release_seq_cst_cmpxchg(
9119; GFX6-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9120; GFX6:       ; %bb.0: ; %entry
9121; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
9122; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
9123; GFX6-NEXT:    s_mov_b32 m0, -1
9124; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9125; GFX6-NEXT:    v_mov_b32_e32 v0, s2
9126; GFX6-NEXT:    v_mov_b32_e32 v1, s1
9127; GFX6-NEXT:    v_mov_b32_e32 v2, s0
9128; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9129; GFX6-NEXT:    s_endpgm
9130;
9131; GFX7-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9132; GFX7:       ; %bb.0: ; %entry
9133; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9134; GFX7-NEXT:    s_mov_b32 m0, -1
9135; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9136; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9137; GFX7-NEXT:    v_mov_b32_e32 v1, s2
9138; GFX7-NEXT:    v_mov_b32_e32 v2, s1
9139; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9140; GFX7-NEXT:    s_endpgm
9141;
9142; GFX10-WGP-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9143; GFX10-WGP:       ; %bb.0: ; %entry
9144; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9145; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9146; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9147; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
9148; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
9149; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9150; GFX10-WGP-NEXT:    s_endpgm
9151;
9152; GFX10-CU-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9153; GFX10-CU:       ; %bb.0: ; %entry
9154; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9155; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9156; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9157; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9158; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9159; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9160; GFX10-CU-NEXT:    s_endpgm
9161;
9162; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9163; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9164; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9165; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
9166; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9167; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9168; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
9169; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
9170; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9171; SKIP-CACHE-INV-NEXT:    s_endpgm
9172;
9173; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9174; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9175; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9176; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9177; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9178; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9179; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9180; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9181; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9182;
9183; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9184; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9185; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9186; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9187; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9188; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9189; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9190; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9191; GFX90A-TGSPLIT-NEXT:    s_endpgm
9192;
9193; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9194; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9195; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9196; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9197; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9198; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9199; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9200; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9201; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9202;
9203; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9204; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9205; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9206; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9207; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9208; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9209; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9210; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9211; GFX940-TGSPLIT-NEXT:    s_endpgm
9212;
9213; GFX11-WGP-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9214; GFX11-WGP:       ; %bb.0: ; %entry
9215; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9216; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9217; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9218; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
9219; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9220; GFX11-WGP-NEXT:    s_endpgm
9221;
9222; GFX11-CU-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
9223; GFX11-CU:       ; %bb.0: ; %entry
9224; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9225; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9226; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9227; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9228; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9229; GFX11-CU-NEXT:    s_endpgm
9230    i32 addrspace(3)* %out, i32 %in, i32 %old) {
9231entry:
9232  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
9233  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release seq_cst
9234  ret void
9235}
9236
9237define amdgpu_kernel void @local_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
9238; GFX6-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9239; GFX6:       ; %bb.0: ; %entry
9240; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
9241; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
9242; GFX6-NEXT:    s_mov_b32 m0, -1
9243; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9244; GFX6-NEXT:    v_mov_b32_e32 v0, s2
9245; GFX6-NEXT:    v_mov_b32_e32 v1, s1
9246; GFX6-NEXT:    v_mov_b32_e32 v2, s0
9247; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9248; GFX6-NEXT:    s_endpgm
9249;
9250; GFX7-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9251; GFX7:       ; %bb.0: ; %entry
9252; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9253; GFX7-NEXT:    s_mov_b32 m0, -1
9254; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9255; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9256; GFX7-NEXT:    v_mov_b32_e32 v1, s2
9257; GFX7-NEXT:    v_mov_b32_e32 v2, s1
9258; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9259; GFX7-NEXT:    s_endpgm
9260;
9261; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9262; GFX10-WGP:       ; %bb.0: ; %entry
9263; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9264; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9265; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9266; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
9267; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
9268; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9269; GFX10-WGP-NEXT:    s_endpgm
9270;
9271; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9272; GFX10-CU:       ; %bb.0: ; %entry
9273; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9274; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9275; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9276; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9277; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9278; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9279; GFX10-CU-NEXT:    s_endpgm
9280;
9281; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9282; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9283; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9284; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
9285; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9286; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9287; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
9288; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
9289; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9290; SKIP-CACHE-INV-NEXT:    s_endpgm
9291;
9292; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9293; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9294; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9295; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9296; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9297; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9298; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9299; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9300; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9301;
9302; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9303; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9304; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9305; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9306; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9307; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9308; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9309; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9310; GFX90A-TGSPLIT-NEXT:    s_endpgm
9311;
9312; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9313; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9314; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9315; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9316; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9317; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9318; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9319; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9320; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9321;
9322; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9323; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9324; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9325; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9326; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9327; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9328; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9329; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9330; GFX940-TGSPLIT-NEXT:    s_endpgm
9331;
9332; GFX11-WGP-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9333; GFX11-WGP:       ; %bb.0: ; %entry
9334; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9335; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9336; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9337; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
9338; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9339; GFX11-WGP-NEXT:    s_endpgm
9340;
9341; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
9342; GFX11-CU:       ; %bb.0: ; %entry
9343; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9344; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9345; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9346; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9347; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9348; GFX11-CU-NEXT:    s_endpgm
9349    i32 addrspace(3)* %out, i32 %in, i32 %old) {
9350entry:
9351  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
9352  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel seq_cst
9353  ret void
9354}
9355
9356define amdgpu_kernel void @local_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
9357; GFX6-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9358; GFX6:       ; %bb.0: ; %entry
9359; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
9360; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
9361; GFX6-NEXT:    s_mov_b32 m0, -1
9362; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9363; GFX6-NEXT:    v_mov_b32_e32 v0, s2
9364; GFX6-NEXT:    v_mov_b32_e32 v1, s1
9365; GFX6-NEXT:    v_mov_b32_e32 v2, s0
9366; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9367; GFX6-NEXT:    s_endpgm
9368;
9369; GFX7-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9370; GFX7:       ; %bb.0: ; %entry
9371; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9372; GFX7-NEXT:    s_mov_b32 m0, -1
9373; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9374; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9375; GFX7-NEXT:    v_mov_b32_e32 v1, s2
9376; GFX7-NEXT:    v_mov_b32_e32 v2, s1
9377; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9378; GFX7-NEXT:    s_endpgm
9379;
9380; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9381; GFX10-WGP:       ; %bb.0: ; %entry
9382; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9383; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9384; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9385; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
9386; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
9387; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9388; GFX10-WGP-NEXT:    s_endpgm
9389;
9390; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9391; GFX10-CU:       ; %bb.0: ; %entry
9392; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9393; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9394; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9395; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9396; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9397; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9398; GFX10-CU-NEXT:    s_endpgm
9399;
9400; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9401; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9402; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9403; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
9404; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9405; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9406; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
9407; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
9408; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9409; SKIP-CACHE-INV-NEXT:    s_endpgm
9410;
9411; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9412; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9413; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9414; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9415; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9416; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9417; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9418; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9419; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9420;
9421; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9422; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9423; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9424; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9425; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9426; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9427; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9428; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9429; GFX90A-TGSPLIT-NEXT:    s_endpgm
9430;
9431; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9432; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9433; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9434; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9435; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9436; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9437; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9438; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9439; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9440;
9441; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9442; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9443; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9444; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9445; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9446; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9447; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9448; GFX940-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
9449; GFX940-TGSPLIT-NEXT:    s_endpgm
9450;
9451; GFX11-WGP-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9452; GFX11-WGP:       ; %bb.0: ; %entry
9453; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9454; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9455; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9456; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
9457; GFX11-WGP-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9458; GFX11-WGP-NEXT:    s_endpgm
9459;
9460; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
9461; GFX11-CU:       ; %bb.0: ; %entry
9462; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9463; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9464; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9465; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9466; GFX11-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
9467; GFX11-CU-NEXT:    s_endpgm
9468    i32 addrspace(3)* %out, i32 %in, i32 %old) {
9469entry:
9470  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
9471  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
9472  ret void
9473}
9474
9475define amdgpu_kernel void @local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg(
9476; GFX6-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9477; GFX6:       ; %bb.0: ; %entry
9478; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
9479; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
9480; GFX6-NEXT:    s_mov_b32 m0, -1
9481; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9482; GFX6-NEXT:    v_mov_b32_e32 v0, s2
9483; GFX6-NEXT:    v_mov_b32_e32 v1, s1
9484; GFX6-NEXT:    v_mov_b32_e32 v2, s0
9485; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9486; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9487; GFX6-NEXT:    ds_write_b32 v0, v1
9488; GFX6-NEXT:    s_endpgm
9489;
9490; GFX7-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9491; GFX7:       ; %bb.0: ; %entry
9492; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9493; GFX7-NEXT:    s_mov_b32 m0, -1
9494; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9495; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9496; GFX7-NEXT:    v_mov_b32_e32 v1, s2
9497; GFX7-NEXT:    v_mov_b32_e32 v2, s1
9498; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9499; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9500; GFX7-NEXT:    ds_write_b32 v0, v1
9501; GFX7-NEXT:    s_endpgm
9502;
9503; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9504; GFX10-WGP:       ; %bb.0: ; %entry
9505; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9506; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9507; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9508; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
9509; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
9510; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9511; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9512; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
9513; GFX10-WGP-NEXT:    s_endpgm
9514;
9515; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9516; GFX10-CU:       ; %bb.0: ; %entry
9517; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9518; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9519; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9520; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9521; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9522; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9523; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9524; GFX10-CU-NEXT:    ds_write_b32 v0, v1
9525; GFX10-CU-NEXT:    s_endpgm
9526;
9527; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9528; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9529; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9530; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
9531; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9532; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9533; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
9534; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
9535; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9536; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9537; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
9538; SKIP-CACHE-INV-NEXT:    s_endpgm
9539;
9540; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9541; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9542; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9543; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9544; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9545; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9546; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9547; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9548; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9549; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
9550; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9551;
9552; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9553; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9554; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9555; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9556; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9557; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9558; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9559; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9560; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9561; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
9562; GFX90A-TGSPLIT-NEXT:    s_endpgm
9563;
9564; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9565; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9566; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9567; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9568; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9569; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9570; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9571; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9572; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9573; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
9574; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9575;
9576; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9577; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9578; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9579; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9580; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9581; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9582; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9583; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9584; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9585; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
9586; GFX940-TGSPLIT-NEXT:    s_endpgm
9587;
9588; GFX11-WGP-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9589; GFX11-WGP:       ; %bb.0: ; %entry
9590; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9591; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9592; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9593; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
9594; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
9595; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9596; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
9597; GFX11-WGP-NEXT:    s_endpgm
9598;
9599; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
9600; GFX11-CU:       ; %bb.0: ; %entry
9601; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9602; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9603; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9604; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9605; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
9606; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9607; GFX11-CU-NEXT:    ds_store_b32 v0, v1
9608; GFX11-CU-NEXT:    s_endpgm
9609    i32 addrspace(3)* %out, i32 %in, i32 %old) {
9610entry:
9611  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
9612  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
9613  %val0 = extractvalue { i32, i1 } %val, 0
9614  store i32 %val0, i32 addrspace(3)* %out, align 4
9615  ret void
9616}
9617
9618define amdgpu_kernel void @local_singlethread_one_as_acquire_monotonic_ret_cmpxchg(
9619; GFX6-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9620; GFX6:       ; %bb.0: ; %entry
9621; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
9622; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
9623; GFX6-NEXT:    s_mov_b32 m0, -1
9624; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9625; GFX6-NEXT:    v_mov_b32_e32 v0, s2
9626; GFX6-NEXT:    v_mov_b32_e32 v1, s1
9627; GFX6-NEXT:    v_mov_b32_e32 v2, s0
9628; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9629; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9630; GFX6-NEXT:    ds_write_b32 v0, v1
9631; GFX6-NEXT:    s_endpgm
9632;
9633; GFX7-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9634; GFX7:       ; %bb.0: ; %entry
9635; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9636; GFX7-NEXT:    s_mov_b32 m0, -1
9637; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9638; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9639; GFX7-NEXT:    v_mov_b32_e32 v1, s2
9640; GFX7-NEXT:    v_mov_b32_e32 v2, s1
9641; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9642; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9643; GFX7-NEXT:    ds_write_b32 v0, v1
9644; GFX7-NEXT:    s_endpgm
9645;
9646; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9647; GFX10-WGP:       ; %bb.0: ; %entry
9648; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9649; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9650; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9651; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
9652; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
9653; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9654; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9655; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
9656; GFX10-WGP-NEXT:    s_endpgm
9657;
9658; GFX10-CU-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9659; GFX10-CU:       ; %bb.0: ; %entry
9660; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9661; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9662; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9663; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9664; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9665; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9666; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9667; GFX10-CU-NEXT:    ds_write_b32 v0, v1
9668; GFX10-CU-NEXT:    s_endpgm
9669;
9670; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9671; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9672; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9673; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
9674; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9675; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9676; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
9677; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
9678; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9679; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9680; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
9681; SKIP-CACHE-INV-NEXT:    s_endpgm
9682;
9683; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9684; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9685; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9686; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9687; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9688; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9689; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9690; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9691; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9692; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
9693; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9694;
9695; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9696; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9697; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9698; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9699; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9700; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9701; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9702; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9703; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9704; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
9705; GFX90A-TGSPLIT-NEXT:    s_endpgm
9706;
9707; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9708; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9709; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9710; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9711; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9712; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9713; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9714; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9715; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9716; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
9717; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9718;
9719; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9720; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9721; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9722; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9723; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9724; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9725; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9726; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9727; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9728; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
9729; GFX940-TGSPLIT-NEXT:    s_endpgm
9730;
9731; GFX11-WGP-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9732; GFX11-WGP:       ; %bb.0: ; %entry
9733; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9734; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9735; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9736; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
9737; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
9738; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9739; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
9740; GFX11-WGP-NEXT:    s_endpgm
9741;
9742; GFX11-CU-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
9743; GFX11-CU:       ; %bb.0: ; %entry
9744; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9745; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9746; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9747; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9748; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
9749; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9750; GFX11-CU-NEXT:    ds_store_b32 v0, v1
9751; GFX11-CU-NEXT:    s_endpgm
9752    i32 addrspace(3)* %out, i32 %in, i32 %old) {
9753entry:
9754  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
9755  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
9756  %val0 = extractvalue { i32, i1 } %val, 0
9757  store i32 %val0, i32 addrspace(3)* %out, align 4
9758  ret void
9759}
9760
9761define amdgpu_kernel void @local_singlethread_one_as_release_monotonic_ret_cmpxchg(
9762; GFX6-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
9763; GFX6:       ; %bb.0: ; %entry
9764; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
9765; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
9766; GFX6-NEXT:    s_mov_b32 m0, -1
9767; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9768; GFX6-NEXT:    v_mov_b32_e32 v0, s2
9769; GFX6-NEXT:    v_mov_b32_e32 v1, s1
9770; GFX6-NEXT:    v_mov_b32_e32 v2, s0
9771; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9772; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9773; GFX6-NEXT:    ds_write_b32 v0, v1
9774; GFX6-NEXT:    s_endpgm
9775;
9776; GFX7-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
9777; GFX7:       ; %bb.0: ; %entry
9778; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9779; GFX7-NEXT:    s_mov_b32 m0, -1
9780; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9781; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9782; GFX7-NEXT:    v_mov_b32_e32 v1, s2
9783; GFX7-NEXT:    v_mov_b32_e32 v2, s1
9784; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9785; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9786; GFX7-NEXT:    ds_write_b32 v0, v1
9787; GFX7-NEXT:    s_endpgm
9788;
9789; GFX10-WGP-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
9790; GFX10-WGP:       ; %bb.0: ; %entry
9791; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9792; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9793; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9794; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
9795; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
9796; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9797; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9798; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
9799; GFX10-WGP-NEXT:    s_endpgm
9800;
9801; GFX10-CU-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
9802; GFX10-CU:       ; %bb.0: ; %entry
9803; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9804; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9805; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9806; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9807; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9808; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9809; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9810; GFX10-CU-NEXT:    ds_write_b32 v0, v1
9811; GFX10-CU-NEXT:    s_endpgm
9812;
9813; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
9814; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9815; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9816; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
9817; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9818; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9819; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
9820; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
9821; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9822; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9823; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
9824; SKIP-CACHE-INV-NEXT:    s_endpgm
9825;
9826; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
9827; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9828; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9829; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9830; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9831; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9832; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9833; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9834; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9835; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
9836; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9837;
9838; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
9839; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9840; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9841; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9842; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9843; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9844; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9845; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9846; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9847; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
9848; GFX90A-TGSPLIT-NEXT:    s_endpgm
9849;
9850; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
9851; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9852; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9853; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9854; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9855; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9856; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9857; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9858; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9859; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
9860; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9861;
9862; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
9863; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9864; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9865; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9866; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9867; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9868; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9869; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9870; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9871; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
9872; GFX940-TGSPLIT-NEXT:    s_endpgm
9873;
9874; GFX11-WGP-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
9875; GFX11-WGP:       ; %bb.0: ; %entry
9876; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9877; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9878; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9879; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
9880; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
9881; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9882; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
9883; GFX11-WGP-NEXT:    s_endpgm
9884;
9885; GFX11-CU-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
9886; GFX11-CU:       ; %bb.0: ; %entry
9887; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
9888; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9889; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9890; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
9891; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
9892; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9893; GFX11-CU-NEXT:    ds_store_b32 v0, v1
9894; GFX11-CU-NEXT:    s_endpgm
9895    i32 addrspace(3)* %out, i32 %in, i32 %old) {
9896entry:
9897  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
9898  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
9899  %val0 = extractvalue { i32, i1 } %val, 0
9900  store i32 %val0, i32 addrspace(3)* %out, align 4
9901  ret void
9902}
9903
9904define amdgpu_kernel void @local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg(
9905; GFX6-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
9906; GFX6:       ; %bb.0: ; %entry
9907; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
9908; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
9909; GFX6-NEXT:    s_mov_b32 m0, -1
9910; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9911; GFX6-NEXT:    v_mov_b32_e32 v0, s2
9912; GFX6-NEXT:    v_mov_b32_e32 v1, s1
9913; GFX6-NEXT:    v_mov_b32_e32 v2, s0
9914; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9915; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9916; GFX6-NEXT:    ds_write_b32 v0, v1
9917; GFX6-NEXT:    s_endpgm
9918;
9919; GFX7-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
9920; GFX7:       ; %bb.0: ; %entry
9921; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9922; GFX7-NEXT:    s_mov_b32 m0, -1
9923; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9924; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9925; GFX7-NEXT:    v_mov_b32_e32 v1, s2
9926; GFX7-NEXT:    v_mov_b32_e32 v2, s1
9927; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9928; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9929; GFX7-NEXT:    ds_write_b32 v0, v1
9930; GFX7-NEXT:    s_endpgm
9931;
9932; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
9933; GFX10-WGP:       ; %bb.0: ; %entry
9934; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9935; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9936; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9937; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
9938; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
9939; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9940; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9941; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
9942; GFX10-WGP-NEXT:    s_endpgm
9943;
9944; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
9945; GFX10-CU:       ; %bb.0: ; %entry
9946; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9947; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9948; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9949; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
9950; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
9951; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9952; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9953; GFX10-CU-NEXT:    ds_write_b32 v0, v1
9954; GFX10-CU-NEXT:    s_endpgm
9955;
9956; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
9957; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9958; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9959; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
9960; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9961; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9962; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
9963; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
9964; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9965; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9966; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
9967; SKIP-CACHE-INV-NEXT:    s_endpgm
9968;
9969; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
9970; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9971; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9972; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9973; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9974; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9975; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9976; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9977; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9978; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
9979; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9980;
9981; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
9982; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9983; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
9984; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9985; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9986; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9987; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
9988; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
9989; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9990; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
9991; GFX90A-TGSPLIT-NEXT:    s_endpgm
9992;
9993; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
9994; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9995; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
9996; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9997; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
9998; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
9999; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10000; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10001; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10002; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10003; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10004;
10005; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10006; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10007; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10008; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10009; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10010; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10011; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10012; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10013; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10014; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10015; GFX940-TGSPLIT-NEXT:    s_endpgm
10016;
10017; GFX11-WGP-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10018; GFX11-WGP:       ; %bb.0: ; %entry
10019; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10020; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10021; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10022; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
10023; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10024; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10025; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
10026; GFX11-WGP-NEXT:    s_endpgm
10027;
10028; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10029; GFX11-CU:       ; %bb.0: ; %entry
10030; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10031; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10032; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10033; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10034; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10035; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10036; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10037; GFX11-CU-NEXT:    s_endpgm
10038    i32 addrspace(3)* %out, i32 %in, i32 %old) {
10039entry:
10040  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
10041  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
10042  %val0 = extractvalue { i32, i1 } %val, 0
10043  store i32 %val0, i32 addrspace(3)* %out, align 4
10044  ret void
10045}
10046
10047define amdgpu_kernel void @local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg(
10048; GFX6-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10049; GFX6:       ; %bb.0: ; %entry
10050; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
10051; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
10052; GFX6-NEXT:    s_mov_b32 m0, -1
10053; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10054; GFX6-NEXT:    v_mov_b32_e32 v0, s2
10055; GFX6-NEXT:    v_mov_b32_e32 v1, s1
10056; GFX6-NEXT:    v_mov_b32_e32 v2, s0
10057; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10058; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10059; GFX6-NEXT:    ds_write_b32 v0, v1
10060; GFX6-NEXT:    s_endpgm
10061;
10062; GFX7-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10063; GFX7:       ; %bb.0: ; %entry
10064; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10065; GFX7-NEXT:    s_mov_b32 m0, -1
10066; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10067; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10068; GFX7-NEXT:    v_mov_b32_e32 v1, s2
10069; GFX7-NEXT:    v_mov_b32_e32 v2, s1
10070; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10071; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10072; GFX7-NEXT:    ds_write_b32 v0, v1
10073; GFX7-NEXT:    s_endpgm
10074;
10075; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10076; GFX10-WGP:       ; %bb.0: ; %entry
10077; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10078; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10079; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10080; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
10081; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
10082; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10083; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10084; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
10085; GFX10-WGP-NEXT:    s_endpgm
10086;
10087; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10088; GFX10-CU:       ; %bb.0: ; %entry
10089; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10090; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10091; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10092; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10093; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10094; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10095; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10096; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10097; GFX10-CU-NEXT:    s_endpgm
10098;
10099; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10100; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10101; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10102; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
10103; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10104; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
10105; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
10106; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
10107; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10108; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10109; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
10110; SKIP-CACHE-INV-NEXT:    s_endpgm
10111;
10112; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10113; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10114; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10115; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10116; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10117; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10118; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10119; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10120; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10121; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10122; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10123;
10124; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10125; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10126; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10127; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10128; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10129; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10130; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10131; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10132; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10133; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10134; GFX90A-TGSPLIT-NEXT:    s_endpgm
10135;
10136; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10137; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10138; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10139; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10140; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10141; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10142; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10143; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10144; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10145; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10146; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10147;
10148; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10149; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10150; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10151; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10152; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10153; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10154; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10155; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10156; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10157; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10158; GFX940-TGSPLIT-NEXT:    s_endpgm
10159;
10160; GFX11-WGP-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10161; GFX11-WGP:       ; %bb.0: ; %entry
10162; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10163; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10164; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10165; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
10166; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10167; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10168; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
10169; GFX11-WGP-NEXT:    s_endpgm
10170;
10171; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
10172; GFX11-CU:       ; %bb.0: ; %entry
10173; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10174; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10175; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10176; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10177; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10178; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10179; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10180; GFX11-CU-NEXT:    s_endpgm
10181    i32 addrspace(3)* %out, i32 %in, i32 %old) {
10182entry:
10183  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
10184  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
10185  %val0 = extractvalue { i32, i1 } %val, 0
10186  store i32 %val0, i32 addrspace(3)* %out, align 4
10187  ret void
10188}
10189
10190define amdgpu_kernel void @local_singlethread_one_as_monotonic_acquire_ret_cmpxchg(
10191; GFX6-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10192; GFX6:       ; %bb.0: ; %entry
10193; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
10194; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
10195; GFX6-NEXT:    s_mov_b32 m0, -1
10196; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10197; GFX6-NEXT:    v_mov_b32_e32 v0, s2
10198; GFX6-NEXT:    v_mov_b32_e32 v1, s1
10199; GFX6-NEXT:    v_mov_b32_e32 v2, s0
10200; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10201; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10202; GFX6-NEXT:    ds_write_b32 v0, v1
10203; GFX6-NEXT:    s_endpgm
10204;
10205; GFX7-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10206; GFX7:       ; %bb.0: ; %entry
10207; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10208; GFX7-NEXT:    s_mov_b32 m0, -1
10209; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10210; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10211; GFX7-NEXT:    v_mov_b32_e32 v1, s2
10212; GFX7-NEXT:    v_mov_b32_e32 v2, s1
10213; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10214; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10215; GFX7-NEXT:    ds_write_b32 v0, v1
10216; GFX7-NEXT:    s_endpgm
10217;
10218; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10219; GFX10-WGP:       ; %bb.0: ; %entry
10220; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10221; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10222; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10223; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
10224; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
10225; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10226; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10227; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
10228; GFX10-WGP-NEXT:    s_endpgm
10229;
10230; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10231; GFX10-CU:       ; %bb.0: ; %entry
10232; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10233; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10234; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10235; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10236; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10237; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10238; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10239; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10240; GFX10-CU-NEXT:    s_endpgm
10241;
10242; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10243; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10244; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10245; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
10246; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10247; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
10248; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
10249; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
10250; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10251; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10252; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
10253; SKIP-CACHE-INV-NEXT:    s_endpgm
10254;
10255; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10256; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10257; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10258; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10259; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10260; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10261; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10262; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10263; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10264; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10265; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10266;
10267; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10268; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10269; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10270; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10271; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10272; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10273; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10274; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10275; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10276; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10277; GFX90A-TGSPLIT-NEXT:    s_endpgm
10278;
10279; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10280; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10281; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10282; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10283; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10284; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10285; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10286; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10287; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10288; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10289; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10290;
10291; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10292; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10293; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10294; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10295; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10296; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10297; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10298; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10299; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10300; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10301; GFX940-TGSPLIT-NEXT:    s_endpgm
10302;
10303; GFX11-WGP-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10304; GFX11-WGP:       ; %bb.0: ; %entry
10305; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10306; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10307; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10308; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
10309; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10310; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10311; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
10312; GFX11-WGP-NEXT:    s_endpgm
10313;
10314; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
10315; GFX11-CU:       ; %bb.0: ; %entry
10316; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10317; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10318; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10319; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10320; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10321; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10322; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10323; GFX11-CU-NEXT:    s_endpgm
10324    i32 addrspace(3)* %out, i32 %in, i32 %old) {
10325entry:
10326  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
10327  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic acquire
10328  %val0 = extractvalue { i32, i1 } %val, 0
10329  store i32 %val0, i32 addrspace(3)* %out, align 4
10330  ret void
10331}
10332
10333define amdgpu_kernel void @local_singlethread_one_as_acquire_acquire_ret_cmpxchg(
10334; GFX6-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10335; GFX6:       ; %bb.0: ; %entry
10336; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
10337; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
10338; GFX6-NEXT:    s_mov_b32 m0, -1
10339; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10340; GFX6-NEXT:    v_mov_b32_e32 v0, s2
10341; GFX6-NEXT:    v_mov_b32_e32 v1, s1
10342; GFX6-NEXT:    v_mov_b32_e32 v2, s0
10343; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10344; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10345; GFX6-NEXT:    ds_write_b32 v0, v1
10346; GFX6-NEXT:    s_endpgm
10347;
10348; GFX7-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10349; GFX7:       ; %bb.0: ; %entry
10350; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10351; GFX7-NEXT:    s_mov_b32 m0, -1
10352; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10353; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10354; GFX7-NEXT:    v_mov_b32_e32 v1, s2
10355; GFX7-NEXT:    v_mov_b32_e32 v2, s1
10356; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10357; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10358; GFX7-NEXT:    ds_write_b32 v0, v1
10359; GFX7-NEXT:    s_endpgm
10360;
10361; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10362; GFX10-WGP:       ; %bb.0: ; %entry
10363; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10364; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10365; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10366; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
10367; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
10368; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10369; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10370; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
10371; GFX10-WGP-NEXT:    s_endpgm
10372;
10373; GFX10-CU-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10374; GFX10-CU:       ; %bb.0: ; %entry
10375; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10376; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10377; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10378; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10379; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10380; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10381; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10382; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10383; GFX10-CU-NEXT:    s_endpgm
10384;
10385; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10386; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10387; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10388; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
10389; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10390; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
10391; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
10392; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
10393; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10394; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10395; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
10396; SKIP-CACHE-INV-NEXT:    s_endpgm
10397;
10398; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10399; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10400; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10401; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10402; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10403; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10404; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10405; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10406; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10407; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10408; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10409;
10410; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10411; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10412; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10413; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10414; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10415; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10416; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10417; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10418; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10419; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10420; GFX90A-TGSPLIT-NEXT:    s_endpgm
10421;
10422; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10423; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10424; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10425; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10426; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10427; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10428; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10429; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10430; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10431; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10432; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10433;
10434; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10435; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10436; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10437; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10438; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10439; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10440; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10441; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10442; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10443; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10444; GFX940-TGSPLIT-NEXT:    s_endpgm
10445;
10446; GFX11-WGP-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10447; GFX11-WGP:       ; %bb.0: ; %entry
10448; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10449; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10450; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10451; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
10452; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10453; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10454; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
10455; GFX11-WGP-NEXT:    s_endpgm
10456;
10457; GFX11-CU-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
10458; GFX11-CU:       ; %bb.0: ; %entry
10459; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10460; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10461; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10462; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10463; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10464; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10465; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10466; GFX11-CU-NEXT:    s_endpgm
10467    i32 addrspace(3)* %out, i32 %in, i32 %old) {
10468entry:
10469  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
10470  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
10471  %val0 = extractvalue { i32, i1 } %val, 0
10472  store i32 %val0, i32 addrspace(3)* %out, align 4
10473  ret void
10474}
10475
10476define amdgpu_kernel void @local_singlethread_one_as_release_acquire_ret_cmpxchg(
10477; GFX6-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10478; GFX6:       ; %bb.0: ; %entry
10479; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
10480; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
10481; GFX6-NEXT:    s_mov_b32 m0, -1
10482; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10483; GFX6-NEXT:    v_mov_b32_e32 v0, s2
10484; GFX6-NEXT:    v_mov_b32_e32 v1, s1
10485; GFX6-NEXT:    v_mov_b32_e32 v2, s0
10486; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10487; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10488; GFX6-NEXT:    ds_write_b32 v0, v1
10489; GFX6-NEXT:    s_endpgm
10490;
10491; GFX7-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10492; GFX7:       ; %bb.0: ; %entry
10493; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10494; GFX7-NEXT:    s_mov_b32 m0, -1
10495; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10496; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10497; GFX7-NEXT:    v_mov_b32_e32 v1, s2
10498; GFX7-NEXT:    v_mov_b32_e32 v2, s1
10499; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10500; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10501; GFX7-NEXT:    ds_write_b32 v0, v1
10502; GFX7-NEXT:    s_endpgm
10503;
10504; GFX10-WGP-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10505; GFX10-WGP:       ; %bb.0: ; %entry
10506; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10507; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10508; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10509; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
10510; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
10511; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10512; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10513; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
10514; GFX10-WGP-NEXT:    s_endpgm
10515;
10516; GFX10-CU-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10517; GFX10-CU:       ; %bb.0: ; %entry
10518; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10519; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10520; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10521; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10522; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10523; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10524; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10525; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10526; GFX10-CU-NEXT:    s_endpgm
10527;
10528; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10529; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10530; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10531; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
10532; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10533; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
10534; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
10535; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
10536; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10537; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10538; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
10539; SKIP-CACHE-INV-NEXT:    s_endpgm
10540;
10541; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10542; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10543; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10544; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10545; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10546; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10547; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10548; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10549; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10550; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10551; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10552;
10553; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10554; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10555; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10556; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10557; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10558; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10559; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10560; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10561; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10562; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10563; GFX90A-TGSPLIT-NEXT:    s_endpgm
10564;
10565; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10566; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10567; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10568; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10569; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10570; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10571; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10572; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10573; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10574; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10575; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10576;
10577; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10578; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10579; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10580; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10581; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10582; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10583; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10584; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10585; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10586; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10587; GFX940-TGSPLIT-NEXT:    s_endpgm
10588;
10589; GFX11-WGP-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10590; GFX11-WGP:       ; %bb.0: ; %entry
10591; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10592; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10593; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10594; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
10595; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10596; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10597; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
10598; GFX11-WGP-NEXT:    s_endpgm
10599;
10600; GFX11-CU-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
10601; GFX11-CU:       ; %bb.0: ; %entry
10602; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10603; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10604; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10605; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10606; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10607; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10608; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10609; GFX11-CU-NEXT:    s_endpgm
10610    i32 addrspace(3)* %out, i32 %in, i32 %old) {
10611entry:
10612  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
10613  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
10614  %val0 = extractvalue { i32, i1 } %val, 0
10615  store i32 %val0, i32 addrspace(3)* %out, align 4
10616  ret void
10617}
10618
10619define amdgpu_kernel void @local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
10620; GFX6-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10621; GFX6:       ; %bb.0: ; %entry
10622; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
10623; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
10624; GFX6-NEXT:    s_mov_b32 m0, -1
10625; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10626; GFX6-NEXT:    v_mov_b32_e32 v0, s2
10627; GFX6-NEXT:    v_mov_b32_e32 v1, s1
10628; GFX6-NEXT:    v_mov_b32_e32 v2, s0
10629; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10630; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10631; GFX6-NEXT:    ds_write_b32 v0, v1
10632; GFX6-NEXT:    s_endpgm
10633;
10634; GFX7-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10635; GFX7:       ; %bb.0: ; %entry
10636; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10637; GFX7-NEXT:    s_mov_b32 m0, -1
10638; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10639; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10640; GFX7-NEXT:    v_mov_b32_e32 v1, s2
10641; GFX7-NEXT:    v_mov_b32_e32 v2, s1
10642; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10643; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10644; GFX7-NEXT:    ds_write_b32 v0, v1
10645; GFX7-NEXT:    s_endpgm
10646;
10647; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10648; GFX10-WGP:       ; %bb.0: ; %entry
10649; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10650; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10651; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10652; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
10653; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
10654; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10655; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10656; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
10657; GFX10-WGP-NEXT:    s_endpgm
10658;
10659; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10660; GFX10-CU:       ; %bb.0: ; %entry
10661; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10662; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10663; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10664; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10665; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10666; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10667; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10668; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10669; GFX10-CU-NEXT:    s_endpgm
10670;
10671; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10672; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10673; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10674; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
10675; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10676; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
10677; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
10678; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
10679; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10680; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10681; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
10682; SKIP-CACHE-INV-NEXT:    s_endpgm
10683;
10684; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10685; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10686; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10687; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10688; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10689; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10690; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10691; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10692; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10693; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10694; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10695;
10696; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10697; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10698; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10699; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10700; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10701; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10702; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10703; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10704; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10705; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10706; GFX90A-TGSPLIT-NEXT:    s_endpgm
10707;
10708; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10709; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10710; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10711; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10712; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10713; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10714; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10715; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10716; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10717; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10718; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10719;
10720; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10721; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10722; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10723; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10724; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10725; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10726; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10727; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10728; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10729; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10730; GFX940-TGSPLIT-NEXT:    s_endpgm
10731;
10732; GFX11-WGP-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10733; GFX11-WGP:       ; %bb.0: ; %entry
10734; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10735; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10736; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10737; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
10738; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10739; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10740; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
10741; GFX11-WGP-NEXT:    s_endpgm
10742;
10743; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
10744; GFX11-CU:       ; %bb.0: ; %entry
10745; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10746; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10747; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10748; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10749; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10750; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10751; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10752; GFX11-CU-NEXT:    s_endpgm
10753    i32 addrspace(3)* %out, i32 %in, i32 %old) {
10754entry:
10755  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
10756  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
10757  %val0 = extractvalue { i32, i1 } %val, 0
10758  store i32 %val0, i32 addrspace(3)* %out, align 4
10759  ret void
10760}
10761
10762define amdgpu_kernel void @local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
10763; GFX6-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
10764; GFX6:       ; %bb.0: ; %entry
10765; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
10766; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
10767; GFX6-NEXT:    s_mov_b32 m0, -1
10768; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10769; GFX6-NEXT:    v_mov_b32_e32 v0, s2
10770; GFX6-NEXT:    v_mov_b32_e32 v1, s1
10771; GFX6-NEXT:    v_mov_b32_e32 v2, s0
10772; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10773; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10774; GFX6-NEXT:    ds_write_b32 v0, v1
10775; GFX6-NEXT:    s_endpgm
10776;
10777; GFX7-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
10778; GFX7:       ; %bb.0: ; %entry
10779; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10780; GFX7-NEXT:    s_mov_b32 m0, -1
10781; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10782; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10783; GFX7-NEXT:    v_mov_b32_e32 v1, s2
10784; GFX7-NEXT:    v_mov_b32_e32 v2, s1
10785; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10786; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10787; GFX7-NEXT:    ds_write_b32 v0, v1
10788; GFX7-NEXT:    s_endpgm
10789;
10790; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
10791; GFX10-WGP:       ; %bb.0: ; %entry
10792; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10793; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10794; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10795; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
10796; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
10797; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10798; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10799; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
10800; GFX10-WGP-NEXT:    s_endpgm
10801;
10802; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
10803; GFX10-CU:       ; %bb.0: ; %entry
10804; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10805; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10806; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10807; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10808; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10809; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10810; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10811; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10812; GFX10-CU-NEXT:    s_endpgm
10813;
10814; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
10815; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10816; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10817; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
10818; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10819; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
10820; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
10821; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
10822; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10823; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10824; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
10825; SKIP-CACHE-INV-NEXT:    s_endpgm
10826;
10827; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
10828; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10829; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10830; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10831; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10832; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10833; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10834; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10835; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10836; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10837; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10838;
10839; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
10840; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10841; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10842; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10843; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10844; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10845; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10846; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10847; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10848; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10849; GFX90A-TGSPLIT-NEXT:    s_endpgm
10850;
10851; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
10852; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10853; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10854; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10855; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10856; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10857; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10858; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10859; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10860; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10861; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10862;
10863; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
10864; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10865; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10866; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10867; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10868; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10869; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10870; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10871; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10872; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10873; GFX940-TGSPLIT-NEXT:    s_endpgm
10874;
10875; GFX11-WGP-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
10876; GFX11-WGP:       ; %bb.0: ; %entry
10877; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10878; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10879; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10880; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
10881; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10882; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10883; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
10884; GFX11-WGP-NEXT:    s_endpgm
10885;
10886; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
10887; GFX11-CU:       ; %bb.0: ; %entry
10888; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
10889; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10890; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10891; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
10892; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
10893; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10894; GFX11-CU-NEXT:    ds_store_b32 v0, v1
10895; GFX11-CU-NEXT:    s_endpgm
10896    i32 addrspace(3)* %out, i32 %in, i32 %old) {
10897entry:
10898  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
10899  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
10900  %val0 = extractvalue { i32, i1 } %val, 0
10901  store i32 %val0, i32 addrspace(3)* %out, align 4
10902  ret void
10903}
10904
10905define amdgpu_kernel void @local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg(
10906; GFX6-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
10907; GFX6:       ; %bb.0: ; %entry
10908; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
10909; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
10910; GFX6-NEXT:    s_mov_b32 m0, -1
10911; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10912; GFX6-NEXT:    v_mov_b32_e32 v0, s2
10913; GFX6-NEXT:    v_mov_b32_e32 v1, s1
10914; GFX6-NEXT:    v_mov_b32_e32 v2, s0
10915; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10916; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10917; GFX6-NEXT:    ds_write_b32 v0, v1
10918; GFX6-NEXT:    s_endpgm
10919;
10920; GFX7-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
10921; GFX7:       ; %bb.0: ; %entry
10922; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10923; GFX7-NEXT:    s_mov_b32 m0, -1
10924; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10925; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10926; GFX7-NEXT:    v_mov_b32_e32 v1, s2
10927; GFX7-NEXT:    v_mov_b32_e32 v2, s1
10928; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10929; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10930; GFX7-NEXT:    ds_write_b32 v0, v1
10931; GFX7-NEXT:    s_endpgm
10932;
10933; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
10934; GFX10-WGP:       ; %bb.0: ; %entry
10935; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10936; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10937; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10938; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
10939; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
10940; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10941; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10942; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
10943; GFX10-WGP-NEXT:    s_endpgm
10944;
10945; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
10946; GFX10-CU:       ; %bb.0: ; %entry
10947; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10948; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10949; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10950; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
10951; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
10952; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10953; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10954; GFX10-CU-NEXT:    ds_write_b32 v0, v1
10955; GFX10-CU-NEXT:    s_endpgm
10956;
10957; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
10958; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10959; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10960; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
10961; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10962; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
10963; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
10964; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
10965; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10966; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10967; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
10968; SKIP-CACHE-INV-NEXT:    s_endpgm
10969;
10970; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
10971; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10972; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10973; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10974; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10975; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10976; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10977; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10978; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10979; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
10980; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10981;
10982; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
10983; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10984; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
10985; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10986; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10987; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
10988; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
10989; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
10990; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10991; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
10992; GFX90A-TGSPLIT-NEXT:    s_endpgm
10993;
10994; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
10995; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10996; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
10997; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10998; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
10999; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11000; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11001; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11002; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11003; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11004; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11005;
11006; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
11007; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11008; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11009; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11010; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11011; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11012; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11013; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11014; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11015; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11016; GFX940-TGSPLIT-NEXT:    s_endpgm
11017;
11018; GFX11-WGP-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
11019; GFX11-WGP:       ; %bb.0: ; %entry
11020; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11021; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11022; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11023; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
11024; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11025; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11026; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
11027; GFX11-WGP-NEXT:    s_endpgm
11028;
11029; GFX11-CU-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
11030; GFX11-CU:       ; %bb.0: ; %entry
11031; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11032; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11033; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11034; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
11035; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11036; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11037; GFX11-CU-NEXT:    ds_store_b32 v0, v1
11038; GFX11-CU-NEXT:    s_endpgm
11039    i32 addrspace(3)* %out, i32 %in, i32 %old) {
11040entry:
11041  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
11042  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic seq_cst
11043  %val0 = extractvalue { i32, i1 } %val, 0
11044  store i32 %val0, i32 addrspace(3)* %out, align 4
11045  ret void
11046}
11047
11048define amdgpu_kernel void @local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg(
11049; GFX6-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11050; GFX6:       ; %bb.0: ; %entry
11051; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
11052; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
11053; GFX6-NEXT:    s_mov_b32 m0, -1
11054; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11055; GFX6-NEXT:    v_mov_b32_e32 v0, s2
11056; GFX6-NEXT:    v_mov_b32_e32 v1, s1
11057; GFX6-NEXT:    v_mov_b32_e32 v2, s0
11058; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11059; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11060; GFX6-NEXT:    ds_write_b32 v0, v1
11061; GFX6-NEXT:    s_endpgm
11062;
11063; GFX7-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11064; GFX7:       ; %bb.0: ; %entry
11065; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11066; GFX7-NEXT:    s_mov_b32 m0, -1
11067; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11068; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11069; GFX7-NEXT:    v_mov_b32_e32 v1, s2
11070; GFX7-NEXT:    v_mov_b32_e32 v2, s1
11071; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11072; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11073; GFX7-NEXT:    ds_write_b32 v0, v1
11074; GFX7-NEXT:    s_endpgm
11075;
11076; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11077; GFX10-WGP:       ; %bb.0: ; %entry
11078; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11079; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11080; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11081; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
11082; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
11083; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11084; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11085; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
11086; GFX10-WGP-NEXT:    s_endpgm
11087;
11088; GFX10-CU-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11089; GFX10-CU:       ; %bb.0: ; %entry
11090; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11091; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11092; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11093; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
11094; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
11095; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11096; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11097; GFX10-CU-NEXT:    ds_write_b32 v0, v1
11098; GFX10-CU-NEXT:    s_endpgm
11099;
11100; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11101; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11102; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11103; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
11104; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11105; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
11106; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
11107; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
11108; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11109; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11110; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
11111; SKIP-CACHE-INV-NEXT:    s_endpgm
11112;
11113; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11114; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11115; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11116; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11117; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11118; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11119; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11120; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11121; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11122; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11123; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11124;
11125; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11126; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11127; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11128; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11129; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11130; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11131; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11132; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11133; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11134; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11135; GFX90A-TGSPLIT-NEXT:    s_endpgm
11136;
11137; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11138; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11139; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11140; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11141; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11142; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11143; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11144; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11145; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11146; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11147; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11148;
11149; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11150; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11151; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11152; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11153; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11154; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11155; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11156; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11157; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11158; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11159; GFX940-TGSPLIT-NEXT:    s_endpgm
11160;
11161; GFX11-WGP-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11162; GFX11-WGP:       ; %bb.0: ; %entry
11163; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11164; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11165; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11166; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
11167; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11168; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11169; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
11170; GFX11-WGP-NEXT:    s_endpgm
11171;
11172; GFX11-CU-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
11173; GFX11-CU:       ; %bb.0: ; %entry
11174; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11175; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11176; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11177; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
11178; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11179; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11180; GFX11-CU-NEXT:    ds_store_b32 v0, v1
11181; GFX11-CU-NEXT:    s_endpgm
11182    i32 addrspace(3)* %out, i32 %in, i32 %old) {
11183entry:
11184  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
11185  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire seq_cst
11186  %val0 = extractvalue { i32, i1 } %val, 0
11187  store i32 %val0, i32 addrspace(3)* %out, align 4
11188  ret void
11189}
11190
11191define amdgpu_kernel void @local_singlethread_one_as_release_seq_cst_ret_cmpxchg(
11192; GFX6-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11193; GFX6:       ; %bb.0: ; %entry
11194; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
11195; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
11196; GFX6-NEXT:    s_mov_b32 m0, -1
11197; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11198; GFX6-NEXT:    v_mov_b32_e32 v0, s2
11199; GFX6-NEXT:    v_mov_b32_e32 v1, s1
11200; GFX6-NEXT:    v_mov_b32_e32 v2, s0
11201; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11202; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11203; GFX6-NEXT:    ds_write_b32 v0, v1
11204; GFX6-NEXT:    s_endpgm
11205;
11206; GFX7-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11207; GFX7:       ; %bb.0: ; %entry
11208; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11209; GFX7-NEXT:    s_mov_b32 m0, -1
11210; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11211; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11212; GFX7-NEXT:    v_mov_b32_e32 v1, s2
11213; GFX7-NEXT:    v_mov_b32_e32 v2, s1
11214; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11215; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11216; GFX7-NEXT:    ds_write_b32 v0, v1
11217; GFX7-NEXT:    s_endpgm
11218;
11219; GFX10-WGP-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11220; GFX10-WGP:       ; %bb.0: ; %entry
11221; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11222; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11223; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11224; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
11225; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
11226; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11227; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11228; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
11229; GFX10-WGP-NEXT:    s_endpgm
11230;
11231; GFX10-CU-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11232; GFX10-CU:       ; %bb.0: ; %entry
11233; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11234; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11235; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11236; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
11237; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
11238; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11239; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11240; GFX10-CU-NEXT:    ds_write_b32 v0, v1
11241; GFX10-CU-NEXT:    s_endpgm
11242;
11243; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11244; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11245; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11246; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
11247; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11248; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
11249; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
11250; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
11251; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11252; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11253; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
11254; SKIP-CACHE-INV-NEXT:    s_endpgm
11255;
11256; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11257; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11258; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11259; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11260; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11261; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11262; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11263; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11264; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11265; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11266; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11267;
11268; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11269; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11270; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11271; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11272; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11273; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11274; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11275; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11276; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11277; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11278; GFX90A-TGSPLIT-NEXT:    s_endpgm
11279;
11280; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11281; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11282; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11283; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11284; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11285; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11286; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11287; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11288; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11289; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11290; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11291;
11292; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11293; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11294; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11295; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11296; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11297; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11298; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11299; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11300; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11301; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11302; GFX940-TGSPLIT-NEXT:    s_endpgm
11303;
11304; GFX11-WGP-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11305; GFX11-WGP:       ; %bb.0: ; %entry
11306; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11307; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11308; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11309; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
11310; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11311; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11312; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
11313; GFX11-WGP-NEXT:    s_endpgm
11314;
11315; GFX11-CU-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
11316; GFX11-CU:       ; %bb.0: ; %entry
11317; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11318; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11319; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11320; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
11321; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11322; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11323; GFX11-CU-NEXT:    ds_store_b32 v0, v1
11324; GFX11-CU-NEXT:    s_endpgm
11325    i32 addrspace(3)* %out, i32 %in, i32 %old) {
11326entry:
11327  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
11328  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release seq_cst
11329  %val0 = extractvalue { i32, i1 } %val, 0
11330  store i32 %val0, i32 addrspace(3)* %out, align 4
11331  ret void
11332}
11333
11334define amdgpu_kernel void @local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg(
11335; GFX6-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11336; GFX6:       ; %bb.0: ; %entry
11337; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
11338; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
11339; GFX6-NEXT:    s_mov_b32 m0, -1
11340; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11341; GFX6-NEXT:    v_mov_b32_e32 v0, s2
11342; GFX6-NEXT:    v_mov_b32_e32 v1, s1
11343; GFX6-NEXT:    v_mov_b32_e32 v2, s0
11344; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11345; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11346; GFX6-NEXT:    ds_write_b32 v0, v1
11347; GFX6-NEXT:    s_endpgm
11348;
11349; GFX7-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11350; GFX7:       ; %bb.0: ; %entry
11351; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11352; GFX7-NEXT:    s_mov_b32 m0, -1
11353; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11354; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11355; GFX7-NEXT:    v_mov_b32_e32 v1, s2
11356; GFX7-NEXT:    v_mov_b32_e32 v2, s1
11357; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11358; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11359; GFX7-NEXT:    ds_write_b32 v0, v1
11360; GFX7-NEXT:    s_endpgm
11361;
11362; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11363; GFX10-WGP:       ; %bb.0: ; %entry
11364; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11365; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11366; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11367; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
11368; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
11369; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11370; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11371; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
11372; GFX10-WGP-NEXT:    s_endpgm
11373;
11374; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11375; GFX10-CU:       ; %bb.0: ; %entry
11376; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11377; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11378; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11379; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
11380; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
11381; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11382; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11383; GFX10-CU-NEXT:    ds_write_b32 v0, v1
11384; GFX10-CU-NEXT:    s_endpgm
11385;
11386; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11387; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11388; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11389; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
11390; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11391; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
11392; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
11393; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
11394; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11395; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11396; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
11397; SKIP-CACHE-INV-NEXT:    s_endpgm
11398;
11399; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11400; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11401; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11402; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11403; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11404; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11405; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11406; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11407; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11408; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11409; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11410;
11411; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11412; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11413; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11414; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11415; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11416; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11417; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11418; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11419; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11420; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11421; GFX90A-TGSPLIT-NEXT:    s_endpgm
11422;
11423; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11424; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11425; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11426; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11427; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11428; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11429; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11430; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11431; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11432; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11433; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11434;
11435; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11436; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11437; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11438; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11439; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11440; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11441; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11442; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11443; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11444; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11445; GFX940-TGSPLIT-NEXT:    s_endpgm
11446;
11447; GFX11-WGP-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11448; GFX11-WGP:       ; %bb.0: ; %entry
11449; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11450; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11451; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11452; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
11453; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11454; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11455; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
11456; GFX11-WGP-NEXT:    s_endpgm
11457;
11458; GFX11-CU-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
11459; GFX11-CU:       ; %bb.0: ; %entry
11460; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11461; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11462; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11463; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
11464; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11465; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11466; GFX11-CU-NEXT:    ds_store_b32 v0, v1
11467; GFX11-CU-NEXT:    s_endpgm
11468    i32 addrspace(3)* %out, i32 %in, i32 %old) {
11469entry:
11470  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
11471  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel seq_cst
11472  %val0 = extractvalue { i32, i1 } %val, 0
11473  store i32 %val0, i32 addrspace(3)* %out, align 4
11474  ret void
11475}
11476
11477define amdgpu_kernel void @local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
11478; GFX6-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11479; GFX6:       ; %bb.0: ; %entry
11480; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
11481; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
11482; GFX6-NEXT:    s_mov_b32 m0, -1
11483; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11484; GFX6-NEXT:    v_mov_b32_e32 v0, s2
11485; GFX6-NEXT:    v_mov_b32_e32 v1, s1
11486; GFX6-NEXT:    v_mov_b32_e32 v2, s0
11487; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11488; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11489; GFX6-NEXT:    ds_write_b32 v0, v1
11490; GFX6-NEXT:    s_endpgm
11491;
11492; GFX7-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11493; GFX7:       ; %bb.0: ; %entry
11494; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11495; GFX7-NEXT:    s_mov_b32 m0, -1
11496; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11497; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11498; GFX7-NEXT:    v_mov_b32_e32 v1, s2
11499; GFX7-NEXT:    v_mov_b32_e32 v2, s1
11500; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11501; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11502; GFX7-NEXT:    ds_write_b32 v0, v1
11503; GFX7-NEXT:    s_endpgm
11504;
11505; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11506; GFX10-WGP:       ; %bb.0: ; %entry
11507; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11508; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11509; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11510; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
11511; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
11512; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11513; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11514; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
11515; GFX10-WGP-NEXT:    s_endpgm
11516;
11517; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11518; GFX10-CU:       ; %bb.0: ; %entry
11519; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11520; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11521; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11522; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
11523; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
11524; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11525; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11526; GFX10-CU-NEXT:    ds_write_b32 v0, v1
11527; GFX10-CU-NEXT:    s_endpgm
11528;
11529; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11530; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11531; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11532; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
11533; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11534; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
11535; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
11536; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
11537; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11538; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11539; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
11540; SKIP-CACHE-INV-NEXT:    s_endpgm
11541;
11542; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11543; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11544; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11545; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11546; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11547; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11548; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11549; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11550; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11551; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11552; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11553;
11554; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11555; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11556; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
11557; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11558; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11559; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11560; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11561; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11562; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11563; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11564; GFX90A-TGSPLIT-NEXT:    s_endpgm
11565;
11566; GFX940-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11567; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11568; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11569; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11570; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11571; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11572; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11573; GFX940-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11574; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11575; GFX940-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
11576; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11577;
11578; GFX940-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11579; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11580; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
11581; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11582; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
11583; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
11584; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
11585; GFX940-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
11586; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11587; GFX940-TGSPLIT-NEXT:    ds_write_b32 v0, v1
11588; GFX940-TGSPLIT-NEXT:    s_endpgm
11589;
11590; GFX11-WGP-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11591; GFX11-WGP:       ; %bb.0: ; %entry
11592; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11593; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11594; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11595; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s2
11596; GFX11-WGP-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11597; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11598; GFX11-WGP-NEXT:    ds_store_b32 v0, v1
11599; GFX11-WGP-NEXT:    s_endpgm
11600;
11601; GFX11-CU-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
11602; GFX11-CU:       ; %bb.0: ; %entry
11603; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
11604; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11605; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11606; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s2
11607; GFX11-CU-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
11608; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11609; GFX11-CU-NEXT:    ds_store_b32 v0, v1
11610; GFX11-CU-NEXT:    s_endpgm
11611    i32 addrspace(3)* %out, i32 %in, i32 %old) {
11612entry:
11613  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
11614  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
11615  %val0 = extractvalue { i32, i1 } %val, 0
11616  store i32 %val0, i32 addrspace(3)* %out, align 4
11617  ret void
11618}
11619
11620